Merge pull request #59 from HumanCellAtlas/se-enable-caas

Allow lira to use cromwell-as-a-service
HumanCellAtlas · Apr 19, 2018 · 6e39af0 · 6e39af0
2 parents af81cac + a9d1a43
commit 6e39af0
Show file tree

Hide file tree

Showing 14 changed files with 172 additions and 51 deletions.
diff --git a/kubernetes/caas_login.sh b/kubernetes/caas_login.sh
@@ -10,7 +10,7 @@ FIRECLOUD_CONTACT_EMAIL=$5  # A contact email to use for the FireCloud service a
 FIRECLOUD_GROUP_NAME=$6  # The name of the user group to create in FireCloud to control workflow collection permissions (e.g. "write-access")
 VAULT_TOKEN_FILE=${VAULT_TOKEN_FILE:-"$HOME/.vault-token"}
 FIRECLOUD_URL=${FIRECLOUD_URL:-"https://firecloud.dsde-dev.broadinstitute.org"}
-FIRECLOUD_API_URL=${FIRECLOUD_URL:-"https://firecloud-orchestration.dsde-dev.broadinstitute.org"}
+FIRECLOUD_API_URL=${FIRECLOUD_API_URL:-"https://firecloud-orchestration.dsde-dev.broadinstitute.org"}
 SAM_URL=${SAM_URL:-"https://sam.dsde-dev.broadinstitute.org"}
 
 

diff --git a/kubernetes/listener-config.json.ctmpl b/kubernetes/listener-config.json.ctmpl
@@ -1,11 +1,18 @@
 {{with $environment := env "ENV"}}
-{{with $cromwellSecrets := vault (printf "secret/dsde/mint/%s/common/htpasswd" $environment)}}
 {{with $listenerSecret := vault (printf "secret/dsde/mint/%s/listener/listener_secret" $environment)}}
     {
+      {{ if env "USE_CAAS" | parseBool }}
+        "collection_name": "{{ env "COLLECTION_NAME" }}",
+        "cromwell_url": "https://cromwell.caas-dev.broadinstitute.org/api/workflows/v1",
+      {{ else }}
+        {{ with $cromwellSecrets := vault (printf "secret/dsde/mint/%s/common/htpasswd" $environment) }}
+          "cromwell_password": "{{$cromwellSecrets.Data.cromwell_password}}",
+          "cromwell_user": "{{$cromwellSecrets.Data.cromwell_user}}",
+          "cromwell_url": "https://cromwell.mint-{{ env "ENV" }}.broadinstitute.org/api/workflows/v1",
+        {{ end }}
+      {{ end }}
       "MAX_CONTENT_LENGTH": 10000,
-      "cromwell_password": "{{$cromwellSecrets.Data.cromwell_password}}",
-      "cromwell_url": "https://cromwell.mint-{{ env "ENV" }}.broadinstitute.org/api/workflows/v1",
-      "cromwell_user": "{{$cromwellSecrets.Data.cromwell_user}}",
+      "use_caas": "{{ env "USE_CAAS" }}",
       "env": "{{ env "ENV" }}",
       "notification_token": "{{$listenerSecret.Data.notification_token}}",
       "submit_wdl": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/{{ env "SUBMIT_WDL_DIR" }}submit.wdl",
@@ -37,4 +44,3 @@
     }
 {{end}}
 {{end}}
-{{end}}
diff --git a/kubernetes/listener-deployment-setup.sh b/kubernetes/listener-deployment-setup.sh
@@ -5,7 +5,6 @@
 #   - Update the dss storage service white list (https://github.com/HumanCellAtlas/data-store/blob/master/environment#L63)
 #     to include bluebox-subscription-manager@<GCLOUD_PROJECT>.iam.gserviceaccount.com
 
-
 ENV=$1  #name of the deployment environment (e.g. staging)
 GCLOUD_PROJECT=$2  #ID of the gcloud project (e.g. broad-dsde-mint-staging)
 BLUEBOX_SUBSCRIPTION_KEY_DIR=$3  #absolute path of the subscription key
@@ -15,7 +14,13 @@ SS2_PREFIX=$6
 TENX_PREFIX=$7
 LOG_SINK_NAME=$8
 LOG_DESTINATION=$9
+USE_CAAS=${10}
+COLLECTION_NAME=${11:-""}
 VAULT_TOKEN_FILE=${VAULT_TOKEN_FILE:-"$HOME/.vault-token"}
+CROMWELL_URL=https://cromwell.mint-$ENV.broadinstitute.org/api/workflows/v1
+if ${USE_CAAS}; then
+    CROMWELL_URL=https://cromwell.caas-dev.broadinstitute.org/api/workflows/v1
+fi
 
 error=0
 if [ -z $ENV ]; then
@@ -54,17 +59,24 @@ if [ -z $LOG_DESTINATION ]; then
     echo -e "\nYou must specify a log destination of form pubsub.googleapis.com/projects/<logs_project_id>/topics/<topic_name>"
     error=1
 fi
-
+if [ -z $USE_CAAS ]; then
+    echo -e "\nYou must specify whether to use Cromwell-as-a-Service"
+    error=1
+fi
+if [ $USE_CAAS ] && [ -z $COLLECTION_NAME ]; then
+    echo -e "\nYou must specify a collection to use for creating workflows in Cromwell-as-a-Service"
+    error=1
+fi
 if [ $error -eq 1 ]; then
-    echo -e "\nUsage: bash listener-deployment.sh ENV GCLOUD_PROJECT BLUBOX_SUBSCRIPTION_KEY_DIR DSS_URL PIPELINE_TOOLS_PREFIX SS2_PREFIX TENX_PREFIX LOG_SINK_NAME LOG_DESTINATION\n"
+    echo -e "\nUsage: bash listener-deployment.sh ENV GCLOUD_PROJECT BLUBOX_SUBSCRIPTION_KEY_DIR DSS_URL PIPELINE_TOOLS_PREFIX SS2_PREFIX TENX_PREFIX LOG_SINK_NAME LOG_DESTINATION USE_CAAS COLLECTION_NAME\n"
     exit 1
 fi
 
 
 # Set gcloud project
 gcloud config set project ${GCLOUD_PROJECT}
 
-# Create service account & key
+# Create bluebox service account & key
 echo "Creating bluebox-subscription-manager service account and key"
 KEY_FILE_PATH=${BLUEBOX_SUBSCRIPTION_KEY_DIR}/bluebox-subscription-manager-${ENV}-key.json
 gcloud iam service-accounts create bluebox-subscription-manager --display-name=bluebox-subscription-manager
@@ -92,16 +104,19 @@ cd ../kubernetes
 # Generate config
 echo "Creating Lira config"
 docker run -i --rm \
+    -e VAULT_TOKEN=$(cat $VAULT_TOKEN_FILE) \
     -e INPUT_PATH=/working \
     -e OUT_PATH=/working \
     -e ENV=${ENV} \
+    -e USE_CAAS=${USE_CAAS} \
+    -e COLLECTION_NAME=${COLLECTION_NAME} \
+    -e CROMWELL_URL=${CROMWELL_URL} \
     -e NOTIFICATION_TOKEN=${LISTENER_SECRET} \
     -e PIPELINE_TOOLS_PREFIX=${PIPELINE_TOOLS_PREFIX} \
     -e SS2_PREFIX=${SS2_PREFIX} \
     -e SS2_SUBSCRIPTION_ID=${SS2_SUBSCRIPTION_ID} \
     -e TENX_PREFIX=${TENX_PREFIX} \
     -e TENX_SUBSCRIPTION_ID=${TENX_SUBSCRIPTION_ID} \
-    -v ${VAULT_TOKEN_FILE}:/root/.vault-token \
     -v ${PWD}:/working broadinstitute/dsde-toolbox:k8s \
     /usr/local/bin/render-ctmpl.sh -k /working/listener-config.json.ctmpl
 

diff --git a/kubernetes/listener-deployment.sh b/kubernetes/listener-deployment.sh
@@ -10,7 +10,12 @@ CERT_DIR=$3
 FULLCHAIN_FILENAME=$4
 PRIVKEY_FILENAME=$5
 DNS_ZONE_NAME=$6
-DOCKER_TAG=$7 #version of the quay.io/humancellatlas/secondary-analysis-lira image to deploy
+DOCKER_TAG=$7 # version of the quay.io/humancellatlas/secondary-analysis-lira image to deploy
+GCLOUD_PROJECT=$8
+USE_CAAS=$9  # whether or not to use cromwell-as-a-service
+WORKFLOW_COLLECTION_ID=${10}  # The name of the workflow-collection to create in SAM (e.g. "dev-workflows")
+FIRECLOUD_CONTACT_EMAIL=${11}  # A contact email to use for the FireCloud service account
+FIRECLOUD_GROUP_NAME=${12}  # The name of the user group to create in FireCloud to control workflow collection permissions (e.g. "write-access")
 
 GET_CERTS=${GET_CERTS:-"false"}
 VAULT_TOKEN_FILE=${VAULT_TOKEN_FILE:-"$HOME/.vault-token"}
@@ -20,7 +25,6 @@ DOMAIN=${DOMAIN:-"broadinstitute.org"}
 PUBLIC_DOMAIN=${PUBLIC_DOMAIN:-"data.humancellatlas.org"}
 PUBLIC_URL=${PUBLIC_URL:-"pipelines.$ENV.$PUBLIC_DOMAIN"}
 
-
 if [ -z $ENV ]; then
     echo -e "\nYou must specify a deployment environment"
     error=1
@@ -42,13 +46,23 @@ elif [ -z $DNS_ZONE_NAME ]; then
 elif [ -z $DOCKER_TAG ]; then
     echo -e "\nYou must specify the lira docker version to deploy"
     error=1
+elif [ -z $GCLOUD_PROJECT ]; then
+    echo -e "\nYou must specify a gcloud project for the deployment"
+    error=1
+elif [ -z $USE_CAAS ]; then
+    echo -e "\nYou must specify whether to use Cromwell-as-a-Service"
+    error=1
 fi
 
 if [ $error -eq 1 ]; then
     echo -e "\nUsage: bash listener-deployment.sh ENV VAULT_TOKEN CERT_DIR FULLCHAIN_FILENAME PRIVKEY_FILENAME DNS_ZONE_NAME DOCKER_TAG\n"
     exit 1
 fi
 
+
+#Set gcloud project
+gcloud config set project ${GCLOUD_PROJECT}
+
 # Create listener cluster
 printf "\ncreate listener cluster"
 gcloud container clusters create listener
@@ -110,13 +124,21 @@ aws route53 change-resource-record-sets \
       }]
     }"
 
+
 # Set listener config secret
 printf "\nset listener config secret"
-LIRA_CONFIG_PATH=$CERT_DIR/config.json
-CONFIG_SECRET_NAME=$(bash populate-listener-config-secret.sh $LIRA_CONFIG_PATH)
+CONFIG_FILE=$CERT_DIR/config.json
+
+if $USE_CAAS; then
+    bash caas_login.sh ${GCLOUD_PROJECT} ${ENV} ${CERT_DIR} ${WORKFLOW_COLLECTION_ID} ${FIRECLOUD_CONTACT_EMAIL} ${FIRECLOUD_GROUP_NAME}
+    CAAS_KEY_FILE=$CERT_DIR/caas-${ENV}-key.json
+    CONFIG_SECRET_NAME=$(bash populate-listener-config-secret.sh $CONFIG_FILE $CAAS_KEY_FILE)
+else
+    CONFIG_SECRET_NAME=$(bash populate-listener-config-secret.sh $CONFIG_FILE)
+fi
 
 # Generate listener-deployment.yaml image with the docker image to deploy
-docker run -i --rm -e LIRA_CONFIG=${CONFIG_SECRET_NAME} -e DOCKER_TAG=${DOCKER_TAG} -v ${VAULT_TOKEN_FILE}:/root/.vault-token -v ${PWD}:/working broadinstitute/dsde-toolbox:k8s /usr/local/bin/render-ctmpl.sh -k /working/${KUBE_YAML}.ctmpl
+docker run -i --rm -e LIRA_CONFIG=${CONFIG_SECRET_NAME} -e DOCKER_TAG=${DOCKER_TAG} -e USE_CAAS=${USE_CAAS} -v ${VAULT_TOKEN_FILE}:/root/.vault-token -v ${PWD}:/working broadinstitute/dsde-toolbox:k8s /usr/local/bin/render-ctmpl.sh -k /working/${KUBE_YAML}.ctmpl
 
 # Create deployment
 printf "\ncreating listener deployment"

diff --git a/kubernetes/listener-deployment.yaml.ctmpl b/kubernetes/listener-deployment.yaml.ctmpl
@@ -21,6 +21,10 @@ spec:
         env:
         - name: listener_config
           value: /etc/secondary-analysis/config.json
+        {{ if (env "USE_CAAS") | parseBool }}
+        - name: caas_key
+          value: /etc/secondary-analysis/caas_key.json
+        {{ end }}
         readinessProbe:
           httpGet:
             path: /health
@@ -32,3 +36,7 @@ spec:
           items:
           - key: config
             path: config.json
+          {{ if (env "USE_CAAS") | parseBool }}
+          - key: caas_key
+            path: caas_key.json
+          {{ end }}
diff --git a/kubernetes/populate-listener-config-secret.sh b/kubernetes/populate-listener-config-secret.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 config_file=$1
+caas_key_file=$2
 
 if [ -z $config_file ]; then
     echo -e "\nYou must provide a config file."
@@ -11,6 +12,8 @@ fi
 config_secret_name=listener-config-$(date '+%Y-%m-%d-%H-%M')
 echo $config_secret_name
 
-kubectl create secret generic \
-    $config_secret_name \
-    --from-file=config=$config_file \
+if [ -z $caas_key_file ]; then
+    kubectl create secret generic $config_secret_name --from-file=config=$config_file
+else
+    kubectl create secret generic $config_secret_name --from-file=config=$config_file --from-file=caas_key=$caas_key_file
+fi
diff --git a/lira/api/notifications.py b/lira/api/notifications.py
@@ -1,6 +1,7 @@
 import connexion
-import logging
 import json
+import logging
+import os
 import time
 from flask import current_app
 from cromwell_tools import cromwell_tools
@@ -35,7 +36,7 @@ def post(body):
     logger.info("Preparing to launch {workflow_name} workflow in Cromwell".format(workflow_name=wdl_config.workflow_name))
 
     # Prepare inputs
-    inputs = lira_utils.compose_inputs(wdl_config.workflow_name, uuid, version, lira_config.env)
+    inputs = lira_utils.compose_inputs(wdl_config.workflow_name, uuid, version, lira_config.env, lira_config.use_caas)
     cromwell_inputs_file = json.dumps(inputs)
 
     # Prepare labels
@@ -58,17 +59,30 @@ def post(body):
         }
         status_code = 201
     else:
+        if lira_config.use_caas:
+            options = lira_utils.compose_caas_options(cromwell_submission.options_file, lira_config.env, lira_config.caas_key)
+            options_file = json.dumps(options)
+            auth = {
+                'caas_key': lira_config.caas_key,
+                'collection_name': lira_config.collection_name
+            }
+        else:
+            options_file = cromwell_submission.options_file
+            auth = {
+                'user': lira_config.cromwell_user,
+                'password': lira_config.cromwell_password
+            }
+
         cromwell_response = cromwell_tools.start_workflow(
             wdl_file=cromwell_submission.wdl_file,
             zip_file=cromwell_tools.make_zip_in_memory(cromwell_submission.wdl_deps_dict),
             inputs_file=cromwell_inputs_file,
             inputs_file2=cromwell_submission.wdl_static_inputs_file,
-            options_file=cromwell_submission.options_file,
+            options_file=options_file,
             url=lira_config.cromwell_url,
-            user=lira_config.cromwell_user,
-            password=lira_config.cromwell_password,
             label=cromwell_labels_file,
-            validate_labels=False  # switch off the validators provided by cromwell_tools
+            validate_labels=False,  # switch off the validators provided by cromwell_tools
+            **auth
         )
         if cromwell_response.status_code > 201:
             logger.error("HTTP error content: {content}".format(content=cromwell_response.text))

diff --git a/lira/lira_config.py b/lira/lira_config.py
@@ -2,6 +2,7 @@
 """
 import logging
 import sys
+import os
 from . import lira_utils
 
 
@@ -164,6 +165,20 @@ def __init__(self, config_object, *args, **kwargs):
         logging.getLogger('werkzeug').setLevel(self.log_level_werkzeug)
         logging.getLogger('connexion.decorators.validation').setLevel(self.log_level_connexion_validation)
 
+        # Check cromwell credentials
+        use_caas = config_object.get('use_caas')
+        if not use_caas:
+            config_object['use_caas'] = False
+        if config_object.get('use_caas'):
+            if not config_object.get('collection_name'):
+                self.collection_name = 'lira-{}-workflows'.format(config_object.get('env'))
+            caas_key = os.environ.get('caas_key')
+            if not caas_key:
+                raise ValueError('No service account json key provided for cromwell-as-a-service.')
+            self.caas_key = caas_key
+        elif not config_object.get('cromwell_user') and not config_object.get('cromwell_password'):
+            raise ValueError('User and password required for {}'.format(config_object.get('cromwell_url')))
+
         # parse the wdls section
         wdl_configs = []
         try:
@@ -188,8 +203,7 @@ def required_fields(self):
             'env',
             'submit_wdl',
             'cromwell_url',
-            'cromwell_user',
-            'cromwell_password',
+            'use_caas',
             'notification_token',
             'MAX_CONTENT_LENGTH',
             'wdls'
@@ -210,15 +224,9 @@ def _verify_wdl_configs(wdl_configs):
 
     def __str__(self):
         s = 'LiraConfig({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7})'
-        return s.format(
-            self.env,
-            self.submit_wdl,
-            self.cromwell_url,
-            '(cromwell_user)',
-            '(cromwell_password)',
-            '(notification_token)',
-            self.MAX_CONTENT_LENGTH,
-            self.wdls)
+        return s.format(self.env, self.submit_wdl, self.cromwell_url,
+            ('use_caas: ' + self.use_caas), '(notification_token)',
+            self.MAX_CONTENT_LENGTH,self.wdls)
 
     def __repr__(self):
         s = 'LiraConfig(environment: {0},' \
@@ -236,8 +244,7 @@ def __repr__(self):
             '(cromwell_user)',
             '(cromwell_password)',
             '(notification_token)',
-            self.MAX_CONTENT_LENGTH,
-            self.wdls)
+            self.MAX_CONTENT_LENGTH, self.wdls)
 
 
 class MaxLevelFilter(object):

diff --git a/lira/lira_utils.py b/lira/lira_utils.py
@@ -49,13 +49,14 @@ def extract_uuid_version_subscription_id(msg):
     return uuid, version, subscription_id
 
 
-def compose_inputs(workflow_name, uuid, version, env):
+def compose_inputs(workflow_name, uuid, version, env, use_caas):
     """Create Cromwell inputs file containing bundle uuid and version.
 
     :param str workflow_name: The name of the workflow.
     :param str uuid: uuid of the bundle.
     :param str version: version of the bundle.
     :param str env: runtime environment, such as 'dev', 'staging', 'test' or 'prod'.
+    :param bool use_caas: whether or not to use cromwell-as-a-service
     :return dict: A dictionary of workflow inputs.
     """
     environment = 'integration' if env == 'test' else env
@@ -64,10 +65,34 @@ def compose_inputs(workflow_name, uuid, version, env):
         workflow_name + '.bundle_version': version,
         workflow_name + '.runtime_environment': env,
         workflow_name + '.dss_url': 'https://dss.{}.data.humancellatlas.org/v1'.format(environment),
-        workflow_name + '.submit_url': 'http://api.ingest.{}.data.humancellatlas.org/'.format(environment)
+        workflow_name + '.submit_url': 'http://api.ingest.{}.data.humancellatlas.org/'.format(environment),
+        workflow_name + '.use_caas': use_caas
     }
 
 
+def compose_caas_options(cromwell_options_file, env, caas_key_file=None):
+    """ Append options for using Cromwell-as-a-service to the default options.json file in the wdl config.
+
+    :param str cromwell_options_file: Contents of the options.json file in the wdl config
+    :param str env: runtime environment, such as 'dev', 'staging', 'test' or 'prod'.
+    :param str caas_key_file: Path to the caas_key_file
+    :return dict: A dictionary of workflow outputs.
+    """
+    options_file = cromwell_options_file
+    if isinstance(options_file, bytes):
+        options_file = cromwell_options_file.decode()
+    options_json = json.loads(options_file)
+
+    with open(caas_key_file) as f:
+        caas_key = f.read()
+    options_json.update({
+        'jes_gcs_root': 'gs://broad-dsde-mint-{}-cromwell-execution/caas-cromwell-executions'.format(env),
+        'google_project': 'broad-dsde-mint-{}'.format(env),
+        'user_service_account_json': caas_key
+    })
+    return options_json
+
+
 def parse_github_resource_url(url):
     """Parse a URL which describes a resource file on Github.
 

diff --git a/lira/test/data/config.json b/lira/test/data/config.json
@@ -3,6 +3,7 @@
   "cromwell_url": "https://cromwell.mint-dev.broadinstitute.org/api/workflows/v1",
   "cromwell_password": "test",
   "cromwell_user": "test",
+  "use_caas": false,
   "notification_token": "test",
   "MAX_CONTENT_LENGTH": 10000,
   "submit_wdl": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/v0.1.5/adapter_pipelines/submit.wdl",