Firefox UI Tests #36
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Firefox UI Tests" | |
| on: | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: "0 2 * * *" | |
| env: | |
| browser: firefox-ios | |
| xcode_version: 15.4 | |
| ios_version: 17.5 | |
| ios_simulator_default: iPhone 15 | |
| xcodebuild_scheme: Fennec | |
| xcodebuild_target: Client | |
| test_results_directory: /Users/runner/tmp | |
| jobs: | |
| compile: | |
| name: Compile | |
| runs-on: macos-14 | |
| steps: | |
| - name: Check out source code | |
| uses: actions/[email protected] | |
| - name: Setup Xcode | |
| id: xcode | |
| run: | | |
| sudo rm -rf /Applications/Xcode.app | |
| sudo xcode-select -s /Applications/Xcode_${{ env.xcode_version }}.app/Contents/Developer | |
| xcodebuild -version | |
| ./checkout.sh | |
| ./bootstrap.sh --force | |
| - name: Compile source code | |
| id: compile | |
| run: | | |
| xcodebuild \ | |
| -resolvePackageDependencies \ | |
| -onlyUsePackageVersionsFromResolvedFile | |
| xcodebuild \ | |
| build-for-testing \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ env.ios_simulator_default }},OS=${{ env.ios_version }}' \ | |
| COMPILER_INDEX_STORE_ENABLE=NO CODE_SIGN_IDENTITY= CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO ARCHS="arm64" | |
| working-directory: ${{ env.browser }} | |
| - name: Compress Derived Data | |
| id: compress-dd | |
| run: | | |
| tar czf derived-data.tar.gz ~/DerivedData/Build/Products | |
| - name: Save Derived Data | |
| id: upload-derived-data | |
| uses: actions/[email protected] | |
| with: | |
| name: xcode-cache-deriveddata-${{ github.workflow }}-${{ github.sha }} | |
| path: ./derived-data.tar.gz | |
| retention-days: 2 | |
| run-smoketests: | |
| name: Run smoke tests | |
| runs-on: macos-14 | |
| needs: compile | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ios_simulator: ['iPhone 15 Plus', 'iPad Pro 13-inch (M4)'] | |
| steps: | |
| - name: Check out source code | |
| uses: actions/[email protected] | |
| - name: Install packages | |
| id: packages | |
| run: | | |
| gem install xcpretty -v 0.3.0 | |
| pip install blockkit==1.9.1 | |
| npm install -g [email protected] | |
| - name: Setup Xcode | |
| id: xcode | |
| run: | | |
| sudo rm -rf /Applications/Xcode.app | |
| sudo xcode-select -s /Applications/Xcode_${{ env.xcode_version }}.app/Contents/Developer | |
| xcodebuild -version | |
| ./checkout.sh | |
| ./bootstrap.sh --force | |
| - name: Get derived data | |
| id: download-derived-data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: xcode-cache-deriveddata-${{ github.workflow }}-${{ github.sha }} | |
| - name: Decompress derived data | |
| id: decompress-dd | |
| run: | | |
| tar xzf derived-data.tar.gz -C / | |
| - name: Run Smoketest1 | |
| id: run-smoketest1 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest1 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest1 \ | |
| | tee xcodebuild-smoketest1.log | xcpretty -r junit --output ./junit-smoketest1.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Run Smoketest2 | |
| id: run-smoketest2 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest2 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest2 \ | |
| | tee xcodebuild-smoketest2.log | xcpretty -r junit --output ./junit-smoketest2.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Run Smoketest3 | |
| id: run-smoketest3 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest3 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest3 \ | |
| | tee xcodebuild-smoketest3.log | xcpretty -r junit --output ./junit-smoketest3.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Run Smoketest4 | |
| id: run-smoketest4 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest4 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest4 \ | |
| | tee xcodebuild-smoketest4.log | xcpretty -r junit --output ./junit-smoketest4.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Determine pass/fail status | |
| id: passfail | |
| run: | | |
| echo "Smoketest1 status: "${{ steps.run-smoketest1.outcome }} | |
| echo "Smoketest2 status: "${{ steps.run-smoketest2.outcome }} | |
| echo "Smoketest3 status: "${{ steps.run-smoketest3.outcome }} | |
| echo "Smoketest4 status: "${{ steps.run-smoketest4.outcome }} | |
| if [[ ${{ steps.run-smoketest1.outcome }} != 'success' | |
| || ${{ steps.run-smoketest2.outcome }} != 'success' | |
| || ${{ steps.run-smoketest3.outcome }} != 'success' | |
| || ${{ steps.run-smoketest4.outcome }} != 'success' ]]; then | |
| exit 1 | |
| else | |
| exit 0 | |
| fi | |
| continue-on-error: true | |
| - name: Print test report | |
| id: test-report | |
| run: | | |
| jrm combined.xml junit-*.xml | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --smoke --github --${{ env.browser }} ./combined.xml ./github.md | |
| cat github.md >> $GITHUB_STEP_SUMMARY | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --smoke --slack --${{ env.browser }} ./combined.xml ./slack.json | |
| mv ./combined.xml "junit-smoketests-${{ matrix.ios_simulator }}-`date +"%Y-%m-%d"`.xml" | |
| working-directory: ${{ env.browser }} | |
| - name: Upload junit files | |
| id: upload-junit | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ env.browser }}-smoketests-${{ matrix.ios_simulator }}-junit-${{ github.run_number }} | |
| path: ${{ env.browser }}/junit-smoketests-*.xml | |
| retention-days: 90 | |
| - name: Upload log file | |
| id: upload-log | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ env.browser }}-smoketests-${{ matrix.ios_simulator }}-xcodebuildlog-${{ github.run_number }} | |
| path: ${{ env.browser }}/xcodebuild-smoketest*.log | |
| retention-days: 90 | |
| - name: Report to Slack | |
| id: slack | |
| uses: slackapi/[email protected] | |
| with: | |
| payload-file-path: ${{ env.browser }}/slack.json | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| ios_simulator: ${{ matrix.ios_simulator }} | |
| pass_fail: ${{ steps.passfail.outcome == 'success' && ':white_check_mark:' || ':x:' }} | |
| xcodebuild_test_plan: Smoketests | |
| ref_name: ${{ github.ref_name }} | |
| repository: ${{ github.repository }} | |
| run_id: ${{ github.run_id }} | |
| server_url: ${{ github.server_url }} | |
| sha: ${{ github.sha }} | |
| - name: Return fail status if a test fails | |
| run: | | |
| exit ${{ steps.passfail.outcome == 'success' && '0' || '1' }} | |
| run-fullfunctionaltests: | |
| name: Run full functional tests | |
| runs-on: macos-14 | |
| needs: compile | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ios_simulator: ['iPhone 15 Pro Max'] | |
| steps: | |
| - name: Check out source code | |
| uses: actions/[email protected] | |
| - name: Install packages | |
| id: packages | |
| run: | | |
| gem install xcpretty -v 0.3.0 | |
| pip install blockkit==1.9.1 | |
| - name: Get derived data | |
| id: download-derived-data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: xcode-cache-deriveddata-${{ github.workflow }}-${{ github.sha }} | |
| - name: Decompress derived data | |
| id: decompress-dd | |
| run: | | |
| tar xzf derived-data.tar.gz -C / | |
| - name: Run tests | |
| id: run-tests | |
| run: | | |
| xcrun simctl boot '${{ matrix.ios_simulator }}' | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan FullFunctionalTestPlan \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results \ | |
| | tee xcodebuild.log | xcpretty -r junit && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Print test report | |
| id: test-report | |
| run: | | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --github --${{ env.browser }} ./build/reports/junit.xml ./github.md | |
| cat github.md >> $GITHUB_STEP_SUMMARY | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --slack --${{ env.browser }} ./build/reports/junit.xml ./slack.json | |
| mv ./build/reports/junit.xml "junit-fullfunctional-${{ matrix.ios_simulator }}-`date +"%Y-%m-%d"`.xml" | |
| working-directory: ${{ env.browser }} | |
| - name: Upload junit files | |
| id: upload-junit | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ env.browser }}-fullfunctional-${{ matrix.ios_simulator }}-junit-${{ github.run_number }} | |
| path: ${{ env.browser }}/junit-*.xml | |
| retention-days: 90 | |
| - name: Upload log file | |
| id: upload-log | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ env.browser }}-fullfunctional-${{ matrix.ios_simulator }}-xcodebuildlog-${{ github.run_number }} | |
| path: ${{ env.browser }}/xcodebuild.log | |
| retention-days: 90 | |
| - name: Report to Slack | |
| id: slack | |
| uses: slackapi/[email protected] | |
| with: | |
| payload-file-path: ${{ env.browser }}/slack.json | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| ios_simulator: ${{ matrix.ios_simulator }} | |
| pass_fail: ${{ steps.run-tests.outcome == 'success' && ':white_check_mark:' || ':x:' }} | |
| xcodebuild_test_plan: FullFunctionalTestPlan | |
| ref_name: ${{ github.ref_name }} | |
| repository: ${{ github.repository }} | |
| run_id: ${{ github.run_id }} | |
| server_url: ${{ github.server_url }} | |
| sha: ${{ github.sha }} | |
| - name: Return fail status if a test fails | |
| run: | | |
| exit ${{ steps.run-tests.outcome == 'success' && '0' || '1' }} |