e2ePerformanceTests.yml

name: E2E Performance Tests

on:
  workflow_call:
    inputs:
      PR_NUMBER:
        description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
        type: string
        required: true

  workflow_dispatch:
    inputs:
      PR_NUMBER:
        description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests.
        type: string
        required: true

concurrency:
  group: ${{ github.ref == 'refs/heads/main' && format('{0}-{1}', github.ref, github.sha) || github.ref }}-e2e
  cancel-in-progress: true

jobs:
  prep:
    runs-on: ubuntu-latest
    name: Find the baseline and delta refs, and check for an existing build artifact for that commit
    outputs:
      BASELINE_REF: ${{ steps.getBaselineRef.outputs.BASELINE_REF }}
      DELTA_REF: ${{ steps.getDeltaRef.outputs.DELTA_REF }}
      IS_PR_MERGED: ${{ steps.getPullRequestDetails.outputs.IS_MERGED }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Fetches the entire history

      - name: Fetch main branch
        run: git fetch origin main:main

      - name: Determine "baseline ref" (prev merge commit)
        id: getBaselineRef
        run: |
          # Get the name of the current branch
          current_branch=$(git rev-parse --abbrev-ref HEAD)

          echo "$current_branch"

          if [ "$current_branch" = "main" ]; then
            # On the main branch, find the previous merge commit
            previous_merge=$(git rev-list --merges HEAD~1 | head -n 1)
          else
            # On a feature branch, find the common ancestor of the current branch and main
            previous_merge=$(git merge-base HEAD main)
          fi
          echo "$previous_merge"
          echo "BASELINE_REF=$previous_merge" >> "$GITHUB_OUTPUT"

      - name: Get pull request details
        id: getPullRequestDetails
        uses: ./.github/actions/javascript/getPullRequestDetails
        with:
          GITHUB_TOKEN: ${{ github.token }}
          PULL_REQUEST_NUMBER: ${{ inputs.PR_NUMBER }}
          USER: ${{ github.actor }}

      - name: Determine "delta ref"
        id: getDeltaRef
        run: |
          if [ '${{ steps.getPullRequestDetails.outputs.IS_MERGED }}' == 'true' ]; then
            echo "DELTA_REF=${{ steps.getPullRequestDetails.outputs.MERGE_COMMIT_SHA }}" >> "$GITHUB_OUTPUT"
          else
            # Set dummy git credentials
            git config --global user.email "test@test.com"
            git config --global user.name "Test"

            # Fetch head_ref of unmerged PR
            git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1

            # Merge pull request locally and get merge commit sha
            git merge --allow-unrelated-histories -X ours --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
            git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}

            # Create and push a branch so it can be checked out in another runner
            git checkout -b e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
            git push origin e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}
            echo "DELTA_REF=e2eDelta-${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }}" >> "$GITHUB_OUTPUT"
          fi

  buildBaseline:
    name: Build apk from baseline
    uses: ./.github/workflows/buildAndroid.yml
    needs: prep
    secrets: inherit
    with:
      type: e2e
      ref: ${{ needs.prep.outputs.BASELINE_REF }}
      artifact-prefix: baseline-${{ needs.prep.outputs.BASELINE_REF }}

  buildDelta:
    name: Build apk from delta ref
    uses: ./.github/workflows/buildAndroid.yml
    needs: prep
    secrets: inherit
    with:
      type: e2eDelta
      ref: ${{ needs.prep.outputs.DELTA_REF }}
      artifact-prefix: delta-${{ needs.prep.outputs.DELTA_REF }}

  runTestsInAWS:
    runs-on: ubuntu-latest
    needs: [prep, buildBaseline, buildDelta]
    if: ${{ always() }}
    name: Run E2E tests in AWS device farm
    steps:
      - uses: actions/checkout@v4
        with:
          # The OS_BOTIFY_COMMIT_TOKEN is a personal access token tied to osbotify (we need a PAT to access the artifact API)
          token: ${{ secrets.OS_BOTIFY_COMMIT_TOKEN }}

      - name: Setup Node
        uses: ./.github/actions/composite/setupNode

      - name: Make zip directory for everything to send to AWS Device Farm
        run: mkdir zip

      - name: Download baseline APK
        uses: actions/download-artifact@v4
        id: downloadBaselineAPK
        with:
          name: ${{ needs.buildBaseline.outputs.APK_ARTIFACT_NAME }}
          path: zip

      # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
      - name: Rename baseline APK
        run: mv "${{ steps.downloadBaselineAPK.outputs.download-path }}/app-e2e-release.apk" "${{ steps.downloadBaselineAPK.outputs.download-path }}/app-e2eRelease.apk"

      - name: Download delta APK
        uses: actions/download-artifact@v4
        id: downloadDeltaAPK
        with:
          name: ${{ needs.buildDelta.outputs.APK_ARTIFACT_NAME }}
          path: zip

      - name: Rename delta APK
        run: mv "${{ steps.downloadDeltaAPK.outputs.download-path }}/app-e2edelta-release.apk" "${{ steps.downloadDeltaAPK.outputs.download-path }}/app-e2edeltaRelease.apk"

      - name: Compile test runner to be executable in a nodeJS environment
        run: npm run e2e-test-runner-build

      - name: Copy e2e code into zip folder
        run: cp tests/e2e/dist/index.js zip/testRunner.ts

      - name: Copy profiler binaries into zip folder
        run: cp -r node_modules/@perf-profiler/android/cpp-profiler/bin zip/bin

      - name: Zip everything in the zip directory up
        run: zip -qr App.zip ./zip

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2

      - name: Schedule AWS Device Farm test run on main branch
        uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
        id: schedule-awsdf-main
        with:
          name: App E2E Performance Regression Tests
          project_arn: ${{ secrets.AWS_PROJECT_ARN }}
          device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
          app_file: zip/app-e2eRelease.apk
          app_type: ANDROID_APP
          test_type: APPIUM_NODE
          test_package_file: App.zip
          test_package_type: APPIUM_NODE_TEST_PACKAGE
          test_spec_file: tests/e2e/TestSpec.yml
          test_spec_type: APPIUM_NODE_TEST_SPEC
          remote_src: false
          file_artifacts: |
            Customer Artifacts.zip
            Test spec output.txt
          log_artifacts: debug.log
          cleanup: true
          timeout: 7200

      - name: Print logs if run failed
        if: failure()
        run: |
          echo ${{ steps.schedule-awsdf-main.outputs.data }}
          unzip "Customer Artifacts.zip" -d mainResults
          cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/logcat.txt" || true
          cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log || true
          cat "./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/Test spec output.txt" || true

      - name: Announce failed workflow in Slack
        if: failure()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          custom_payload: |
            {
              channel: '#e2e-announce',
              attachments: [{
                color: 'danger',
                text: `💥 ${process.env.AS_REPO} E2E Test run failed on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow 💥`,
              }]
            }
        env:
          GITHUB_TOKEN: ${{ github.token }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

      - name: Unzip AWS Device Farm results
        if: always()
        run: unzip "Customer Artifacts.zip"

      - name: Print AWS Device Farm run results
        if: always()
        run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"

      - name: Check if test failed, if so post the results and add the DeployBlocker label
        id: checkIfRegressionDetected
        run: |
          if grep -q '🔴' "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"; then
            # Create an output to the GH action that the test failed:
            echo "performanceRegressionDetected=true" >> "$GITHUB_OUTPUT"

            gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
            gh pr comment ${{ inputs.PR_NUMBER }} -F "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"
            gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
          else
            echo "performanceRegressionDetected=false" >> "$GITHUB_OUTPUT"
            echo '✅ no performance regression detected'
          fi
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: Check if test has skipped tests
        id: checkIfSkippedTestsDetected
        run: |
          if grep -q '⚠️' "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"; then
            # Create an output to the GH action that the tests were skipped:
            echo "skippedTestsDetected=true" >> "$GITHUB_OUTPUT"
          else
            echo "skippedTestsDetected=false" >> "$GITHUB_OUTPUT"
            echo '✅ no skipped tests detected'
          fi
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: 'Announce skipped tests in Slack'
        if: ${{ steps.checkIfSkippedTestsDetected.outputs.skippedTestsDetected == 'true' }}
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          custom_payload: |
            {
              channel: '#e2e-announce',
              attachments: [{
                color: 'danger',
                text: `⚠️ ${process.env.AS_REPO} Some of E2E tests were skipped on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow ⚠️`,
              }]
            }
        env:
          GITHUB_TOKEN: ${{ github.token }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

      - name: 'Announce regression in Slack'
        if: ${{ steps.checkIfRegressionDetected.outputs.performanceRegressionDetected == 'true' }}
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          custom_payload: |
            {
              channel: '#quality',
              attachments: [{
                color: 'danger',
                text: `🔴 Performance regression detected in PR ${{ inputs.PR_NUMBER }}\nDetected in <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}> workflow.`,
              }]
            }
        env:
          GITHUB_TOKEN: ${{ github.token }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

  cleanupDeltaRef:
    needs: [prep, runTestsInAWS]
    if: ${{ always() && needs.prep.outputs.IS_PR_MERGED != 'true' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Delete temporary merge branch created for delta ref
        run: git push -d origin ${{ needs.prep.outputs.DELTA_REF }}