-
Notifications
You must be signed in to change notification settings - Fork 7
227 lines (197 loc) · 8.59 KB
/
test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
# This workflow will build a Java project with Gradle
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
name: Run Service Tests
on:
push:
branches:
- main
paths-ignore:
- '*.md'
- '.github/**'
- 'service/local-dev/**'
pull_request:
branches:
- main
# There is an issue with GitHub required checks and paths-ignore. We don't really need to
# run the tests if there are only irrelevant changes (see paths-ignore above). However,
# we require tests to pass by making a "required check" rule on the branch. If the action
# is not triggered, the required check never passes and you are stuck. Therefore, we have
# to run tests even when we only change a markdown file. So don't do what I did and put a
# paths-ignore right here!
workflow_dispatch:
inputs:
testEnv:
description: 'Environment in which tests should be run. Regardless of how this is set, the tests run against a local Postgres and development Sam'
required: true
jobs:
build-test-publish-azureDatabaseUtils:
runs-on: ubuntu-latest
steps:
- name: Checkout current code
uses: actions/checkout@v3
with:
# fetch full history for sonar
fetch-depth: 0
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 17
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v3
- name: Construct docker image name and tag
id: image-name
run: echo name=us.gcr.io/broad-dsp-gcr-public/azure-database-utils:${GITHUB_SHA} >> $GITHUB_OUTPUT
- name: Run tests
run: ./gradlew --build-cache :azureDatabaseUtils:test --scan
- name: Build docker
run: ./gradlew --build-cache :azureDatabaseUtils:jibDockerBuild --image=${{ steps.image-name.outputs.name }} -Djib.console=plain --scan
- name: Run Trivy vulnerability scanner
# Link to the github location of the action https://github.com/broadinstitute/dsp-appsec-trivy-action
uses: broadinstitute/dsp-appsec-trivy-action@v1
with:
image: ${{ steps.image-name.outputs.name }}
- name: Auth to Google
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCR_PUBLISH_KEY }}
- name: Setup gcloud
uses: google-github-actions/setup-gcloud@v1
- name: Explicitly auth Docker for GCR
run: gcloud auth configure-docker --quiet
- name: Push GCR image
run: "docker push ${{ steps.image-name.outputs.name }}"
- name: SonarQube scan
run: ./gradlew --build-cache :azureDatabaseUtils:sonarqube
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
test-job:
runs-on: ubuntu-latest
# the azure connected tests require azureDatabaseUtils to be built and published
needs: build-test-publish-azureDatabaseUtils
# A note on our use of a matrix here:
# Github workflows don't really support reusing code very well. Every workflow runs on a clean
# instance, so we can't share a setup workflow. We could write a custom action, but you can't
# call an action from another action, and they also don't have access to secrets.
# Github also doesn't support yaml anchors (https://github.community/t/support-for-yaml-anchors/),
# so we're using a matrix.
strategy:
fail-fast: false
matrix:
gradleTask: [unitTest, connectedTest, azureUnitTest, azureConnectedTest, awsUnitTest]
steps:
- name: Checkout current code
uses: actions/checkout@v3
with:
# fetch full history for sonar
fetch-depth: 0
- name: Skip version bump merges
id: skiptest
uses: ./.github/actions/bump-skip
with:
event-name: ${{ github.event_name }}
- name: Set env
if: steps.skiptest.outputs.is-bump == 'no'
id: set-env-step
run: |
if ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}; then
ENV=local
elif ${{ github.event_name == 'workflow_dispatch' }}; then
ENV=${{ github.event.inputs.testEnv }}
else
echo ::error ::${{ github.event_name }} not supported for this workflow
exit 1
fi
echo test-env=$ENV >> $GITHUB_OUTPUT
- name: Set up JDK
if: steps.skiptest.outputs.is-bump == 'no'
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 17
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v3
- name: Write config
id: config
uses: ./.github/actions/write-credentials
with:
user-delegated-sa-b64: ${{ secrets.USER_DELEGATED_SA_DEV }}
buffer-client-sa-b64: ${{ secrets.BUFFER_CLIENT_SA_DEV }}
testrunner-sa-b64: ${{ secrets.TESTRUNNER_SA_DEV }}
testrunner-k8s-sa-b64: ${{ secrets.TESTRUNNER_K8S_SA_DEV }}
wsm-sa-b64: ${{ secrets.WSM_SA_DEV }}
janitor-sa-b64: ${{ secrets.JANITOR_SA_DEV }}
policy-client-sa-b64: ${{ secrets.POLICY_CLIENT_SA_DEV }}
- name: Store az creds
id: store-az-creds
run: |
WSM_AZURE_PUBLISHER_CLIENT_ID=${{ secrets.WSM_AZURE_PUBLISHER_CLIENT_ID }}
echo ::add-mask::$WSM_AZURE_PUBLISHER_CLIENT_ID
WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID=${{ secrets.WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID }}
echo ::add-mask::$WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID
WSM_AZURE_PUBLISHER_TENANT_ID=${{ secrets.WSM_AZURE_PUBLISHER_TENANT_ID }}
echo ::add-mask::$WSM_AZURE_PUBLISHER_TENANT_ID
echo wsm-azure-publisher-client-id=$WSM_AZURE_PUBLISHER_CLIENT_ID >> ${GITHUB_OUTPUT}
echo wsm-azure-publisher-client-secret-id=$WSM_AZURE_PUBLISHER_CLIENT_SECRET_ID >> ${GITHUB_OUTPUT}
echo wsm-azure-publisher-tenant-id=$WSM_AZURE_PUBLISHER_TENANT_ID >> ${GITHUB_OUTPUT}
- name: Write config
id: write-config
run: |
cat << EOF > "config/local-properties.yml"
workspace:
policy:
base-path: https://tps.dsde-dev.broadinstitute.org/
cli:
server-name: broad-dev
azure:
managed-app-client-id: "${{ steps.store-az-creds.outputs.wsm-azure-publisher-client-id }}"
managed-app-client-secret: "${{ steps.store-az-creds.outputs.wsm-azure-publisher-client-secret-id }}"
managed-app-tenant-id: "${{ steps.store-az-creds.outputs.wsm-azure-publisher-tenant-id }}"
feature:
tps-enabled: true
temporary-grant-enabled: true
EOF
# Run tests
- name: Run tests
if: steps.skiptest.outputs.is-bump == 'no'
env:
# PRINT_STANDARD_STREAMS is temporary to let us inspect logs for a particular
# issue with Stairway serdes.
PRINT_STANDARD_STREAMS: please
TEST_ENV: ${{ steps.set-env-step.outputs.test-env }}
run: ./gradlew :service:${{ matrix.gradleTask }} --scan
- name: SonarQube scan
if: steps.skiptest.outputs.is-bump == 'no'
run: ./gradlew --build-cache :service:sonarqube
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Notify QA Slack"
if: always() && (steps.set-env-step.outputs.test-env == 'alpha' || steps.set-env-step.outputs.test-env == 'staging')
uses: broadinstitute/[email protected]
# see https://github.com/broadinstitute/action-slack
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
MATRIX_CONTEXT: ${{ toJson(matrix) }}
with:
status: ${{ job.status }}
channel: "#dsde-qa"
username: "Workspace Manager ${{ steps.set-env-step.outputs.test-env }} tests"
author_name: "Workspace Manager ${{ steps.set-env-step.outputs.test-env }} ${{ matrix.gradleTask }}"
fields: repo,job,workflow,commit,eventName,author,took
- name: "Notify WSM Slack"
# post to WSM Slack when a regular push (i.e. non-bumper push) is made to main branch
if: failure() && github.event_name == 'push' && steps.skiptest.outputs.is-bump == 'no'
uses: broadinstitute/[email protected]
# see https://github.com/broadinstitute/action-slack
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
MATRIX_CONTEXT: ${{ toJson(matrix) }}
with:
status: ${{ job.status }}
channel: "#dsp-workspaces-test-alerts"
username: "WSM push to main branch"
author_name: "${{ matrix.gradleTask }}"
icon_emoji: ":triangular_ruler:"
fields: job, commit