Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add make target and ci test for pangolin workflow #1575

Open
wants to merge 3 commits into
base: trunk
Choose a base branch
from
Open

add make target and ci test for pangolin workflow #1575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
de4806c
add make target and test for pangolin
phoenixAja Jan 25, 2023
7765298
pipeline inputs to pangolin
phoenixAja Jan 25, 2023
d8f323a
build pangolin image in CI
phoenixAja Jan 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/push-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ jobs:
docker buildx build --cache-from=type=registry,ref=${{ secrets.ECR_REPO }}/genepi-backend:branch-trunk -t ${{ secrets.ECR_REPO }}/genepi-backend:latest src/backend
make local-init LOCALDEV_PROFILE=backend
make backend-test
gisaid-test:
workflows-gisaid-pangolin-test:
runs-on: ubuntu-20.04
steps:
- name: Configure AWS prod Credentials
Expand All @@ -221,9 +221,12 @@ jobs:
make gha-setup
# Build gisaid image
# docker buildx build --cache-from=type=registry,ref=${{ secrets.ECR_REPO }}/genepi-backend:branch-trunk -t ${{ secrets.ECR_REPO }}/genepi-gisaid:latest -f src/backend/Dockerfile.gisaid src/backend
# Build pangolin image
# docker buildx build --cache-from=type=registry,ref=${{ secrets.ECR_REPO }}/genepi-backend:branch-trunk -t ${{ secrets.ECR_REPO }}/genepi-pangolin:latest -f src/backend/Dockerfile.pangolin src/backend
make local-init LOCALDEV_PROFILE=gisaid
python3 -m pip install miniwdl
make pipeline-test-gisaid
make pipeline-test-pangolin
build-push-images:
if: github.ref == 'refs/heads/trunk'
runs-on: ubuntu-20.04
Expand Down
4 changes: 2 additions & 2 deletions .happy/terraform/modules/sfn_config/pangolin-ondemand.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@ version 1.1

workflow pangolin {
input {
String docker_image_id = "pangolin:latest"
String docker_image_id = "genepi-pangolin:latest"
String aws_region = "us-west-2"
String genepi_config_secret_name
String remote_dev_prefix = ""
Array[String] samples
}

call pangolin_workflow {
input:
docker_image_id = docker_image_id,
Expand All @@ -29,6 +28,7 @@ task pangolin_workflow {
}

command <<<
export AWS_REGION="~{aws_region}"
export GENEPI_CONFIG_SECRET_NAME="~{genepi_config_secret_name}"
if [ "~{remote_dev_prefix}" != "" ]; then
export REMOTE_DEV_PREFIX="~{remote_dev_prefix}"
Expand Down
12 changes: 12 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,18 @@ pipeline-test-gisaid:
miniwdl run --cfg miniwdl.cfg --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env BOTO_ENDPOINT_URL --input test_inputs.json --verbose -o output.json ../../../.happy/terraform/modules/sfn_config/gisaid.wdl
$(docker_compose) run --no-deps --rm backend make pipeline-test-gisaid

.PHONY: pipeline-test-pangolin
pipeline-test-pangolin:
source .env.ecr; \
export DOCKER_REPO; \
export BOTO_ENDPOINT_URL=http://localstack.genepinet.localdev:4566; \
export AWS_ACCESS_KEY_ID=NONCE; \
export AWS_SECRET_ACCESS_KEY=NONCE; \
export MINIWDL_CFG=miniwdl.cfg; \
cd src/backend/pipeline_tests; \
cat test_data/pangolin_pipeline_inputs.json | envsubst > test_inputs.json; \
miniwdl run --debug --cfg miniwdl.cfg --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env BOTO_ENDPOINT_URL --input test_inputs.json --verbose -o output.json ../../../.happy/terraform/modules/sfn_config/pangolin-ondemand.wdl
$(docker_compose) run --no-deps --rm backend make pipeline-test-pangolin

### WDL ###################################################
.PHONY: wdl-lint
Expand Down
6 changes: 6 additions & 0 deletions src/backend/pipeline_tests/test_data/pangolin_pipeline_inputs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"docker_image_id": "${DOCKER_REPO}genepi-pangolin:latest",
"aws_region": "us-west-2",
"genepi_config_secret_name": "genepi-config",
"samples": ["OCPH20", "OCPH21", "OCPH22", "OCPH23", "OCPH24"]
}