Skip to content
Build and Test

DC-1004: Point to new #dsp-data-exploration slack channel (#176) #1393

Sign in to view logs

DC-1004: Point to new #dsp-data-exploration slack channel (#176)

DC-1004: Point to new #dsp-data-exploration slack channel (#176) #1393

Workflow file for this run

.github/workflows/build-and-test.yml at 7ddd6b2
name: Build and Test
on:
push:
branches: [ main ]
paths-ignore: [ '*.md' ]
pull_request:
branches: [ '**' ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install black and link shellcheck into expected location
run: |
pip install black --force-reinstall black==22.3.0
sudo ln -s $(which shellcheck) /usr/local/bin/shellcheck
- name: Set up JDK
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: 'gradle'
- name: Build all projects without running tests
run: ./gradlew --build-cache build -x test
- name: Upload spotbugs results
uses: github/codeql-action/upload-sarif@main
with:
sarif_file: service/build/reports/spotbugs/main.sarif
dispatch-trivy:
needs: [ build ]
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Fire off Trivy action
uses: broadinstitute/workflow-dispatch@v3
with:
workflow: Trivy
token: ${{ secrets.BROADBOT_TOKEN }}
ref: ${{ github.event.pull_request.head.ref }}
source-clear:
needs: [ build ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up JDK
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: 'gradle'
- name: SourceClear scan
env:
SRCCLR_API_TOKEN: ${{ secrets.SRCCLR_API_TOKEN }}
run: ./gradlew --build-cache srcclr
unit-tests-and-sonar:
needs: [ build ]
runs-on: ubuntu-latest
services:
postgres:
image: postgres:13
env:
POSTGRES_PASSWORD: postgres
ports: [ "5432:5432" ]
steps:
- uses: actions/checkout@v3
# Needed by sonar to get the git history for the branch the PR will be merged into.
with:
fetch-depth: 0
- name: Set up JDK
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: 'gradle'
- name: Make sure Postgres is ready and init
env:
PGPASSWORD: postgres
run: |
pg_isready -h localhost -t 10
psql -h localhost -U postgres -f scripts/postgres-init.sql
- name: Test with coverage
run: ./gradlew --build-cache test jacocoTestReport
# The Sonar scan is done here, so it can upload the coverage report generated by the tests.
- name: Sonar scan
run: ./gradlew --build-cache :common:sonar
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
admin-cli-test:
needs: [ build ]
runs-on: ubuntu-latest
services:
postgres:
image: postgres:13
env:
POSTGRES_PASSWORD: postgres
ports: [ "5432:5432" ]
steps:
- uses: actions/checkout@v3
- name: Set up JDK
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: 'gradle'
- name: Make sure Postgres is ready and init
env:
PGPASSWORD: postgres
run: |
pg_isready -h localhost -t 10
psql -h localhost -U postgres -f scripts/postgres-init.sql
- name: Smoke test -- run the CLI
run: UPGRADE_DB=true ./gradlew --build-cache :admin-cli:bootRun --args=list
integration-tests:
needs: [ build ]
runs-on: ubuntu-latest
services:
postgres:
image: postgres:13
env:
POSTGRES_PASSWORD: postgres
ports: [ "5432:5432" ]
steps:
- uses: actions/checkout@v3
- name: Set up JDK
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: 'gradle'
- name: Construct docker image name and tag
id: image-name
run: |
GITHUB_REPO=$(basename ${{ github.repository }})
GIT_SHORT_HASH=$(git rev-parse --short HEAD)
echo "name=${GITHUB_REPO}:${GIT_SHORT_HASH}" >> $GITHUB_OUTPUT
- name: Build image locally with jib
run: |
./gradlew --build-cache :service:jibDockerBuild \
--image=${{ steps.image-name.outputs.name }} \
-Djib.console=plain
- name: Make sure Postgres is ready and init
env:
PGPASSWORD: postgres
run: |
pg_isready -h localhost -t 10
psql -h localhost -U postgres -f scripts/postgres-init.sql
- name: Render GitHub Secrets
run: |
echo "${{ secrets.DEV_FIRECLOUD_ACCOUNT_B64 }}" | base64 -d > "integration/src/main/resources/rendered/user-delegated-sa.json"
echo "${{ secrets.PERF_TESTRUNNER_ACCOUNT_B64 }}" | base64 -d > "integration/src/main/resources/rendered/testrunner-perf.json"
- name: Launch the background process for integration tests
# 172.17.0.1 is the IP address of the gateway in the default docker bridge network. https://docs.docker.com/network/network-tutorial-standalone/
run: docker run -e DATABASE_HOSTNAME=172.17.0.1 -e SPRING_PROFILES_INCLUDE=human-readable-logging -p 8080:8080 -d --name catalog ${{ steps.image-name.outputs.name }}
- name: Wait for boot run to be ready
run: |
set +e
timeout 20 bash -c 'until [[ $(curl -s localhost:8080/status | jq .ok) == "true" ]]; do sleep 1; done'
resultStatus=$?
set -e
if [[ $resultStatus == 0 ]]; then
echo "Server started successfully"
else
echo "Server did not start successfully. Dumping log file"
docker logs catalog
exit 1
fi
- name: Run the integration test suite
run: ./gradlew --build-cache runTest --args="suites/local/FullIntegration.json build/reports"
notify-slack:
needs: [ unit-tests-and-sonar, source-clear, integration-tests, admin-cli-test ]
runs-on: ubuntu-latest
if: failure() && github.ref == 'refs/heads/main'
steps:
- name: Notify slack on failure
uses: broadinstitute/[email protected]
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
with:
channel: '#dsp-data-exploration'
status: failure
author_name: Build on dev
fields: job,message
text: 'Build failed :sadpanda:'
username: 'Data Explorer GitHub Action'
dispatch-tag:
needs: [ unit-tests-and-sonar, source-clear, integration-tests, admin-cli-test ]
runs-on: ubuntu-latest
if: success() && github.ref == 'refs/heads/main'
steps:
- name: Fire off tag action
uses: broadinstitute/workflow-dispatch@v3
with:
workflow: Tag
token: ${{ secrets.BROADBOT_TOKEN }}