[TOAZ-371] Misc fixes for running wsm on azure. #8568
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will build a Java project with Gradle | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle | |
| name: Run Service Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - '*.md' | |
| - '.github/**' | |
| - 'service/local-dev/**' | |
| pull_request: | |
| branches: | |
| - main | |
| # There is an issue with GitHub required checks and paths-ignore. We don't really need to | |
| # run the tests if there are only irrelevant changes (see paths-ignore above). However, | |
| # we require tests to pass by making a "required check" rule on the branch. If the action | |
| # is not triggered, the required check never passes and you are stuck. Therefore, we have | |
| # to run tests even when we only change a markdown file. So don't do what I did and put a | |
| # paths-ignore right here! | |
| workflow_dispatch: | |
| inputs: | |
| testEnv: | |
| description: 'Environment in which tests should be run. Regardless of how this is set, the tests run against a local Postgres and development Sam' | |
| required: true | |
| jobs: | |
| build-test-publish-azureDatabaseUtils: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout current code | |
| uses: actions/checkout@v3 | |
| with: | |
| # fetch full history for sonar | |
| fetch-depth: 0 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Construct docker image name and tag | |
| id: image-name | |
| run: echo name=us.gcr.io/broad-dsp-gcr-public/azure-database-utils:${GITHUB_SHA} >> $GITHUB_OUTPUT | |
| - name: Run tests | |
| run: ./gradlew --build-cache :azureDatabaseUtils:test --scan | |
| - name: Build docker | |
| run: ./gradlew --build-cache :azureDatabaseUtils:jibDockerBuild --image=${{ steps.image-name.outputs.name }} -Djib.console=plain --scan | |
| - name: Run Trivy vulnerability scanner | |
| # Link to the github location of the action https://github.com/broadinstitute/dsp-appsec-trivy-action | |
| uses: broadinstitute/dsp-appsec-trivy-action@v1 | |
| with: | |
| image: ${{ steps.image-name.outputs.name }} | |
| - name: Auth to Google | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCR_PUBLISH_KEY }} | |
| - name: Setup gcloud | |
| uses: google-github-actions/setup-gcloud@v1 | |
| - name: Explicitly auth Docker for GCR | |
| run: gcloud auth configure-docker --quiet | |
| - name: Push GCR image | |
| run: "docker push ${{ steps.image-name.outputs.name }}" | |
| - name: SonarQube scan | |
| run: ./gradlew --build-cache :azureDatabaseUtils:sonarqube | |
| env: | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| test-job: | |
| runs-on: ubuntu-latest | |
| # the azure connected tests require azureDatabaseUtils to be built and published | |
| needs: build-test-publish-azureDatabaseUtils | |
| # A note on our use of a matrix here: | |
| # Github workflows don't really support reusing code very well. Every workflow runs on a clean | |
| # instance, so we can't share a setup workflow. We could write a custom action, but you can't | |
| # call an action from another action, and they also don't have access to secrets. | |
| # Github also doesn't support yaml anchors (https://github.community/t/support-for-yaml-anchors/), | |
| # so we're using a matrix. | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| gradleTask: [unitTest, connectedTest, azureUnitTest, azureConnectedTest, awsUnitTest] | |
| steps: | |
| - name: Checkout current code | |
| uses: actions/checkout@v3 | |
| with: | |
| # fetch full history for sonar | |
| fetch-depth: 0 | |
| - name: Skip version bump merges | |
| id: skiptest | |
| uses: ./.github/actions/bump-skip | |
| with: | |
| event-name: ${{ github.event_name }} | |
| - name: Set env | |
| if: steps.skiptest.outputs.is-bump == 'no' | |
| id: set-env-step | |
| run: | | |
| if ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}; then | |
| ENV=local | |
| elif ${{ github.event_name == 'workflow_dispatch' }}; then | |
| ENV=${{ github.event.inputs.testEnv }} | |
| else | |
| echo ::error ::${{ github.event_name }} not supported for this workflow | |
| exit 1 | |
| fi | |
| echo test-env=$ENV >> $GITHUB_OUTPUT | |
| - name: Set up JDK | |
| if: steps.skiptest.outputs.is-bump == 'no' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Write config | |
| id: config | |
| uses: ./.github/actions/write-credentials | |
| with: | |
| user-delegated-sa-b64: ${{ secrets.USER_DELEGATED_SA_DEV }} | |
| buffer-client-sa-b64: ${{ secrets.BUFFER_CLIENT_SA_DEV }} | |
| testrunner-sa-b64: ${{ secrets.TESTRUNNER_SA_DEV }} | |
| testrunner-k8s-sa-b64: ${{ secrets.TESTRUNNER_K8S_SA_DEV }} | |
| wsm-sa-b64: ${{ secrets.WSM_SA_DEV }} | |
| janitor-sa-b64: ${{ secrets.JANITOR_SA_DEV }} | |
| policy-client-sa-b64: ${{ secrets.POLICY_CLIENT_SA_DEV }} | |
| - name: Store az creds | |
| id: store-az-creds | |
| run: | | |
| WSM_AZURE_PUBLISHER_CLIENT_ID=${{ secrets.WSM_AZURE_PUBLISHER_CLIENT_ID }} | |
| echo ::add-mask::$WSM_AZURE_PUBLISHER_CLIENT_ID | |
| WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID=${{ secrets.WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID }} | |
| echo ::add-mask::$WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID | |
| WSM_AZURE_PUBLISHER_TENANT_ID=${{ secrets.WSM_AZURE_PUBLISHER_TENANT_ID }} | |
| echo ::add-mask::$WSM_AZURE_PUBLISHER_TENANT_ID | |
| echo wsm-azure-publisher-client-id=$WSM_AZURE_PUBLISHER_CLIENT_ID >> ${GITHUB_OUTPUT} | |
| echo wsm-azure-publisher-client-secret-id=$WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID >> ${GITHUB_OUTPUT} | |
| echo wsm-azure-publisher-tenant-id=$WSM_AZURE_PUBLISHER_TENANT_ID >> ${GITHUB_OUTPUT} | |
| - name: Write config | |
| id: write-config | |
| run: | | |
| cat << EOF > "config/local-properties.yml" | |
| workspace: | |
| policy: | |
| base-path: https://tps.dsde-dev.broadinstitute.org/ | |
| cli: | |
| server-name: broad-dev | |
| azure: | |
| managed-app-client-id: "${{ steps.store-az-creds.outputs.wsm-azure-publisher-client-id }}" | |
| managed-app-client-secret: "${{ steps.store-az-creds.outputs.wsm-azure-publisher-client-secret-id }}" | |
| managed-app-tenant-id: "${{ steps.store-az-creds.outputs.wsm-azure-publisher-tenant-id }}" | |
| feature: | |
| tps-enabled: true | |
| temporary-grant-enabled: true | |
| EOF | |
| # Run tests | |
| - name: Run tests | |
| if: steps.skiptest.outputs.is-bump == 'no' | |
| env: | |
| # PRINT_STANDARD_STREAMS is temporary to let us inspect logs for a particular | |
| # issue with Stairway serdes. | |
| PRINT_STANDARD_STREAMS: please | |
| TEST_ENV: ${{ steps.set-env-step.outputs.test-env }} | |
| run: ./gradlew :service:${{ matrix.gradleTask }} --scan | |
| - name: SonarQube scan | |
| if: steps.skiptest.outputs.is-bump == 'no' | |
| run: ./gradlew --build-cache :service:sonarqube | |
| env: | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: "Notify QA Slack" | |
| if: always() && (steps.set-env-step.outputs.test-env == 'alpha' || steps.set-env-step.outputs.test-env == 'staging') | |
| uses: broadinstitute/[email protected] | |
| # see https://github.com/broadinstitute/action-slack | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| MATRIX_CONTEXT: ${{ toJson(matrix) }} | |
| with: | |
| status: ${{ job.status }} | |
| channel: "#dsde-qa" | |
| username: "Workspace Manager ${{ steps.set-env-step.outputs.test-env }} tests" | |
| author_name: "Workspace Manager ${{ steps.set-env-step.outputs.test-env }} ${{ matrix.gradleTask }}" | |
| fields: repo,job,workflow,commit,eventName,author,took | |
| - name: "Notify WSM Slack" | |
| # post to WSM Slack when a regular push (i.e. non-bumper push) is made to main branch | |
| if: failure() && github.event_name == 'push' && steps.skiptest.outputs.is-bump == 'no' | |
| uses: broadinstitute/[email protected] | |
| # see https://github.com/broadinstitute/action-slack | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| MATRIX_CONTEXT: ${{ toJson(matrix) }} | |
| with: | |
| status: ${{ job.status }} | |
| channel: "#dsp-workspaces-test-alerts" | |
| username: "WSM push to main branch" | |
| author_name: "${{ matrix.gradleTask }}" | |
| icon_emoji: ":triangular_ruler:" | |
| fields: job, commit |