From ca9857c7982aa52c99ea265ca66f4aec80b77a28 Mon Sep 17 00:00:00 2001 From: Dave Shiga Date: Fri, 2 Feb 2018 13:56:28 -0500 Subject: [PATCH] Make Lira compatible with Python 3. Add ability to cache WDLs and to run in "dry run" mode. Redact full config from logging. Fix config template. (538) --- .travis.yml | 3 +- Dockerfile | 13 ++- kubernetes/listener-config.json.ctmpl | 16 ++-- lira/api/notifications.py | 132 +++++++++++++++++++------- lira/lira.py | 7 +- lira/lira.yml | 13 +-- lira/{config.py => lira_config.py} | 58 +++++------ lira/lira_utils.py | 26 ++++- lira/test/data/config.json | 4 +- lira/test/test.sh | 2 +- lira/test/test_config.py | 51 ++++++---- lira/test/test_lira_utils.py | 2 +- lira/test/test_notifications.py | 131 +++++++++++++++++++++++++ lira/test/test_service.py | 2 +- 14 files changed, 345 insertions(+), 115 deletions(-) rename lira/{config.py => lira_config.py} (72%) create mode 100644 lira/test/test_notifications.py diff --git a/.travis.yml b/.travis.yml index 2552817e..a25e939e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: python python: - '2.7' -sudo: enabled +- '3.6' +sudo: required services: docker script: cd lira/test && bash test.sh notifications: diff --git a/Dockerfile b/Dockerfile index a5ad92dc..a488fedb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,22 +2,21 @@ FROM ubuntu:16.04 RUN apt-get update && apt-get upgrade -y && \ apt-get -y install --no-install-recommends \ - python-pip \ - python-setuptools \ + python3-pip \ vim \ nmap \ git -RUN pip install --upgrade pip +RUN pip3 install --upgrade pip setuptools -RUN pip install wheel +RUN pip3 install wheel -RUN mkdir /secondary-analysis -WORKDIR /secondary-analysis +RUN mkdir /lira +WORKDIR /lira COPY requirements.txt . -RUN pip install -r requirements.txt +RUN pip3 install -r requirements.txt COPY . . diff --git a/kubernetes/listener-config.json.ctmpl b/kubernetes/listener-config.json.ctmpl index d183d000..8f8e26f3 100644 --- a/kubernetes/listener-config.json.ctmpl +++ b/kubernetes/listener-config.json.ctmpl @@ -1,12 +1,13 @@ {{with $environment := env "ENV"}} {{with $cromwellSecrets := vault (printf "secret/dsde/mint/%s/common/htpasswd" $environment)}} +{{with $listenerSecret := vault (printf "secret/dsde/mint/%s/listener/listener_secret" $environment)}} { "MAX_CONTENT_LENGTH": 10000, "cromwell_password": "{{$cromwellSecrets.Data.cromwell_password}}", "cromwell_url": "https://cromwell.mint-{{ env "ENV" }}.broadinstitute.org/api/workflows/v1", "cromwell_user": "{{$cromwellSecrets.Data.cromwell_user}}", "env": "{{ env "ENV" }}", - "notification_token": "{{env "NOTIFICATION_TOKEN" }}", + "notification_token": "{{$listenerSecret.Data.notification_token}}", "submit_wdl": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/submit.wdl", "wdls": [ { @@ -14,23 +15,23 @@ "{{ env "TENX_PREFIX" }}/pipelines/10x/count/count.wdl" ], "options_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/10x/options.json", - "subscription_id": {{ env "TENX_SUBSCRIPTION_ID" }}, - "wdl_default_inputs_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/10x/adapter_example_static.json", + "subscription_id": "{{ env "TENX_SUBSCRIPTION_ID" }}", + "wdl_static_inputs_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/10x/adapter_example_static.json", "wdl_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/10x/adapter.wdl", "workflow_name": "Adapter10xCount" }, { "analysis_wdls": [ "{{ env "SS2_PREFIX" }}/pipelines/smartseq2_single_sample/ss2_single_sample.wdl", - "{{ env "SS2_PREFIX" }}/library/subworkflows/smartseq2_single_sample/pipelines/hisat2_QC_pipeline.wdl", - "{{ env "SS2_PREFIX" }}/library/subworkflows/smartseq2_single_sample/pipelines/hisat2_rsem_pipeline.wdl", + "{{ env "SS2_PREFIX" }}/library/subworkflows/hisat2_QC_pipeline.wdl", + "{{ env "SS2_PREFIX" }}/library/subworkflows/hisat2_rsem_pipeline.wdl", "{{ env "SS2_PREFIX" }}/library/tasks/hisat2.wdl", "{{ env "SS2_PREFIX" }}/library/tasks/picard.wdl", "{{ env "SS2_PREFIX" }}/library/tasks/rsem.wdl" ], "options_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/ss2_single_sample/options.json", - "subscription_id": {{ env "SS2_SUBSCRIPTION_ID" }}, - "wdl_default_inputs_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/ss2_single_sample/adapter_example_static.json", + "subscription_id": "{{ env "SS2_SUBSCRIPTION_ID" }}", + "wdl_static_inputs_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/ss2_single_sample/adapter_example_static.json", "wdl_link": "{{ env "PIPELINE_TOOLS_PREFIX" }}/adapter_pipelines/ss2_single_sample/adapter.wdl", "workflow_name": "AdapterSmartSeq2SingleCell" } @@ -38,3 +39,4 @@ } {{end}} {{end}} +{{end}} diff --git a/lira/api/notifications.py b/lira/api/notifications.py index b185dcae..e7e99eec 100644 --- a/lira/api/notifications.py +++ b/lira/api/notifications.py @@ -6,13 +6,15 @@ from cromwell_tools import cromwell_tools from lira import lira_utils +logger = logging.getLogger("Lira | {module_path}".format(module_path=__name__)) + def post(body): - # Check authentication - logger = logging.getLogger("Lira | {module_path}".format(module_path=__name__)) - green_config = current_app.config + """Process notification and launch workflow in Cromwell""" - if not lira_utils.is_authenticated(connexion.request.args, green_config.notification_token): + # Check authentication + lira_config = current_app.config + if not lira_utils.is_authenticated(connexion.request.args, lira_config.notification_token): time.sleep(1) return lira_utils.response_with_server_header(dict(error='Unauthorized'), 401) @@ -23,43 +25,101 @@ def post(body): uuid, version, subscription_id = lira_utils.extract_uuid_version_subscription_id(body) # Find wdl config where the subscription_id matches the notification's subscription_id - id_matches = [wdl for wdl in green_config.wdls if wdl.subscription_id == subscription_id] + id_matches = [wdl for wdl in lira_config.wdls if wdl.subscription_id == subscription_id] if len(id_matches) == 0: return lira_utils.response_with_server_header( dict(error='Not Found: No wdl config found with subscription id {}' ''.format(subscription_id)), 404) - wdl = id_matches[0] - logger.debug("Matched WDL: {wdl}".format(wdl=wdl)) - logger.info("Launching {workflow_name} workflow in Cromwell".format(workflow_name=wdl.workflow_name)) + wdl_config = id_matches[0] + logger.debug("Matched WDL config: {wdl}".format(wdl=wdl_config)) + logger.info("Preparing to launch {workflow_name} workflow in Cromwell".format(workflow_name=wdl_config.workflow_name)) # Prepare inputs - inputs = lira_utils.compose_inputs(wdl.workflow_name, uuid, version, green_config.env) + inputs = lira_utils.compose_inputs(wdl_config.workflow_name, uuid, version, lira_config.env) cromwell_inputs_file = json.dumps(inputs) - # Read files into memory - wdl_file = cromwell_tools.download(wdl.wdl_link) - wdl_default_inputs_file = cromwell_tools.download(wdl.wdl_default_inputs_link) - options_file = cromwell_tools.download(wdl.options_link) - - # Create zip of analysis and submit wdls - url_to_contents = cromwell_tools.download_to_map(wdl.analysis_wdls + [green_config.submit_wdl]) - wdl_deps_file = cromwell_tools.make_zip_in_memory(url_to_contents) - - cromwell_response = cromwell_tools.start_workflow( - wdl_file=wdl_file, - zip_file=wdl_deps_file, - inputs_file=cromwell_inputs_file, - inputs_file2=wdl_default_inputs_file, - options_file=options_file, - url=green_config.cromwell_url, - user=green_config.cromwell_user, - password=green_config.cromwell_password - ) - - # Respond - if cromwell_response.status_code > 201: - logger.error("HTTP error content: {content}".format(content=cromwell_response.text)) - return lira_utils.response_with_server_header(dict(result=cromwell_response.text), 500) - logger.info("Cromwell response: {response}".format(response=cromwell_response.json())) - - return lira_utils.response_with_server_header(cromwell_response.json()) + cromwell_submission = current_app.prepare_submission(wdl_config, lira_config.submit_wdl) + logger.info(current_app.prepare_submission.cache_info()) + + + dry_run = getattr(lira_config, 'dry_run', False) + if dry_run: + logger.warning('Not launching workflow because Lira is in dry_run mode') + response_json = { + 'id': 'fake_id', + 'status': 'fake_status' + } + status_code = 201 + else: + cromwell_response = cromwell_tools.start_workflow( + wdl_file=cromwell_submission.wdl_file, + zip_file=cromwell_submission.wdl_deps_file, + inputs_file=cromwell_inputs_file, + inputs_file2=cromwell_submission.wdl_static_inputs_file, + options_file=cromwell_submission.options_file, + url=lira_config.cromwell_url, + user=lira_config.cromwell_user, + password=lira_config.cromwell_password + ) + if cromwell_response.status_code > 201: + logger.error("HTTP error content: {content}".format(content=cromwell_response.text)) + response_json = dict(result=cromwell_response.text) + status_code = 500 + else: + response_json = cromwell_response.json() + logger.info("Cromwell response: {0} {1}".format(response_json, cromwell_response.status_code)) + status_code = 201 + + return lira_utils.response_with_server_header(response_json, status_code) + + +def create_prepare_submission_function(cache_wdls): + """Returns decorated prepare_submission function. Decorator is determined as follows: + Python 2: Always decorate with noop_lru_cache, since functools.lru_cache is not available in 2. + Python 3: Use functools.lru_cache if cache_wdls is true, otherwise use noop_lru_cache.""" + + # Use noop_lru_cache unless cache_wdls is true and functools.lru_cache is available + lru_cache = lira_utils.noop_lru_cache + if not cache_wdls: + logger.info('Not caching wdls because Lira is configured with cache_wdls false') + else: + try: + from functools import lru_cache + except ImportError: + logger.info('Not caching wdls because functools.lru_cache is not available in Python 2.') + + @lru_cache(maxsize=None) + # Setting maxsize to None here means each unique call to prepare_submission (defined by parameters used) + # will be added to the cache without evicting any other items. Additional non-unique calls + # (parameters identical to a previous call) will read from the cache instead of actually + # calling this function. This means that the function will be called no more than once for + # each wdl config, but we can have arbitrarily many wdl configs without running out of space + # in the cache. + def prepare_submission(wdl_config, submit_wdl): + """Load into memory all static data needed for submitting a workflow to Cromwell""" + + # Read files into memory + wdl_file = cromwell_tools.download(wdl_config.wdl_link) + wdl_static_inputs_file = cromwell_tools.download(wdl_config.wdl_static_inputs_link) + options_file = cromwell_tools.download(wdl_config.options_link) + + # Create zip of analysis and submit wdls + url_to_contents = cromwell_tools.download_to_map(wdl_config.analysis_wdls + [submit_wdl]) + wdl_deps_file = cromwell_tools.make_zip_in_memory(url_to_contents) + + return CromwellSubmission(wdl_file, wdl_static_inputs_file, options_file, wdl_deps_file) + + return prepare_submission + + +class CromwellSubmission(object): + """Holds static data needed for submitting a workflow to Cromwell, including + the top level wdl file, the static inputs json file, the options file, + and the dependencies zip file. + """ + + def __init__(self, wdl_file, wdl_static_inputs_file=None, options_file=None, wdl_deps_file=None): + self.wdl_file = wdl_file + self.wdl_static_inputs_file = wdl_static_inputs_file + self.options_file = options_file + self.wdl_deps_file = wdl_deps_file diff --git a/lira/lira.py b/lira/lira.py index e1f775d7..3d7e3885 100755 --- a/lira/lira.py +++ b/lira/lira.py @@ -10,7 +10,8 @@ from connexion.resolver import RestyResolver import argparse -import config +from . import lira_config +from .api import notifications parser = argparse.ArgumentParser() parser.add_argument('--host', default='0.0.0.0') @@ -26,8 +27,10 @@ application = app.app config_path = os.environ['listener_config'] +logger.info('Using config file at {0}'.format(config_path)) with open(config_path) as f: - app.app.config = config.ListenerConfig(json.load(f), app.app.config) + app.app.config = lira_config.LiraConfig(json.load(f), app.app.config) + app.app.prepare_submission = notifications.create_prepare_submission_function(app.app.config.cache_wdls) resolver = RestyResolver('lira.api', collection_endpoint_name='list') app.add_api('lira.yml', resolver=resolver, validate_responses=True) diff --git a/lira/lira.yml b/lira/lira.yml index 6ba8e959..fe71e6b7 100644 --- a/lira/lira.yml +++ b/lira/lira.yml @@ -36,7 +36,7 @@ paths: schema: $ref: '#/definitions/Notification' responses: - 200: + 201: description: OK schema: $ref: '#/definitions/NotificationResponse' @@ -77,13 +77,10 @@ definitions: NotificationResponse: type: object properties: - result: - type: object - properties: - status: - type: string - id: - type: string + status: + type: string + id: + type: string HealthCheckResponse: type: object properties: diff --git a/lira/config.py b/lira/lira_config.py similarity index 72% rename from lira/config.py rename to lira/lira_config.py index 0c0f7649..13327328 100644 --- a/lira/config.py +++ b/lira/lira_config.py @@ -6,7 +6,7 @@ class Config(object): def __init__(self, config_dictionary, flask_config_values=None): - """abstract class that defines some useful configuration checks for the listener + """abstract class that defines some useful configuration checks for Lira This object takes a dictionary of values and verifies them against expected_keys. It additionally accepts values from a flask config object, which it merges @@ -35,17 +35,18 @@ def required_fields(self): def _verify_fields(self): """Verify config contains required fields""" - wdl_keys = set(self.config_dictionary) - extra_keys = wdl_keys - self.required_fields - missing_keys = self.required_fields - wdl_keys + given_keys = set(self.config_dictionary) + extra_keys = given_keys - self.required_fields + missing_keys = self.required_fields - given_keys if missing_keys: raise ValueError( - 'The following configuration is missing key(s): {keys}\n{wdl}' - ''.format(wdl=self.config_dictionary, keys=', '.join(missing_keys))) + 'The following configuration is missing key(s): {keys}' + ''.format(keys=', '.join(missing_keys))) if extra_keys: - logging.warning( - 'The following configuration has unexpected key(s): {keys}\n{wdl}' - ''.format(wdl=self.config_dictionary, keys=', '.join(extra_keys))) + logger = logging.getLogger('Lira | {module_path}'.format(module_path=__name__)) + logger.info( + 'Configuration has non-required key(s): {keys}' + ''.format(keys=', '.join(extra_keys))) def __eq__(self, other): return isinstance(other, WdlConfig) and hash(self) == hash(other) @@ -53,21 +54,16 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) - def to_string(self): + def __str__(self): """recursively collapse a config object into a hashable string""" result = '' for v in self.required_fields: field_value = getattr(self, v) - if isinstance(field_value, (str, unicode, int, list)): - result += str(field_value) - elif isinstance(field_value, Config): - result += field_value.to_string() - else: - raise ValueError('Config fields must either be strings or config objects') + result += str(field_value) return result def __hash__(self): - return hash(self.to_string()) + return hash(str(self)) # needed for flask interface def __getitem__(self, item): @@ -87,7 +83,7 @@ def required_fields(self): 'wdl_link', 'analysis_wdls', 'workflow_name', - 'wdl_default_inputs_link', + 'wdl_static_inputs_link', 'options_link' } @@ -99,24 +95,31 @@ def _verify_fields(self): def __str__(self): s = 'WdlConfig({0}, {1}, {2}, {3}, {4}, {5})' return s.format(self.subscription_id, self.wdl_link, self.analysis_wdls, - self.workflow_name, self.wdl_default_inputs_link, self.options_link) + self.workflow_name, self.wdl_static_inputs_link, self.options_link) + +class LiraConfig(Config): + """subclass of Config representing Lira configuration""" -class ListenerConfig(Config): - """subclass of Config to check listener configurations""" + def __init__(self, config_object, *args, **kwargs): + logger = logging.getLogger('Lira | {module_path}'.format(module_path=__name__)) - def __init__(self, json_config_object, *args, **kwargs): + # Default value that can be overridden + self.cache_wdls = True # parse the wdls section wdl_configs = [] try: - for wdl in json_config_object['wdls']: + for wdl in config_object['wdls']: wdl_configs.append(WdlConfig(wdl)) except KeyError: raise ValueError('supplied config file must contain a "wdls" section.') self._verify_wdl_configs(wdl_configs) - json_config_object['wdls'] = wdl_configs + config_object['wdls'] = wdl_configs + + if config_object.get('dry_run'): + logger.warning('***Lira is running in dry_run mode and will NOT launch any workflows***') - Config.__init__(self, json_config_object, *args, **kwargs) + Config.__init__(self, config_object, *args, **kwargs) @property def required_fields(self): @@ -135,7 +138,8 @@ def required_fields(self): def _verify_wdl_configs(wdl_configs): """Additional verification for wdl configurations""" if len(wdl_configs) != len(set(wdl_configs)): - logging.warning('duplicate wdl definitions detected in config.json') + logger = logging.getLogger('Lira | {module_path}'.format(module_path=__name__)) + logger.warning('duplicate wdl definitions detected in config.json') if len(wdl_configs) != len(set([wdl.subscription_id for wdl in wdl_configs])): raise ValueError( @@ -144,7 +148,7 @@ def _verify_wdl_configs(wdl_configs): 'contents.') def __str__(self): - s = 'ListenerConfig({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7})' + s = 'LiraConfig({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7})' return s.format(self.env, self.submit_wdl, self.cromwell_url, '(cromwell_user)', '(cromwell_password)', '(notification_token)', self.MAX_CONTENT_LENGTH, self.wdls) diff --git a/lira/lira_utils.py b/lira/lira_utils.py index 7d0c4b56..8d4e3ca4 100644 --- a/lira/lira_utils.py +++ b/lira/lira_utils.py @@ -5,7 +5,7 @@ from flask import make_response -def response_with_server_header(body, status=200): +def response_with_server_header(body, status): """Add information of server to header.We are doing this to overwrite the default flask Server header. The default header is a security risk because it provides too much information about server internals. @@ -13,7 +13,7 @@ def response_with_server_header(body, status=200): :param int status: HTTP response status code. :return flask.wrappers.Response response: HTTP response with information of server in header. """ - response = make_response(json.dumps(body), status) + response = make_response(json.dumps(body, indent=2) + '\n', status) response.headers['Server'] = 'Secondary Analysis Service' response.headers['Content-type'] = 'application/json' return response @@ -57,3 +57,25 @@ def compose_inputs(workflow_name, uuid, version, env): workflow_name + '.bundle_version': version, workflow_name + '.runtime_environment': env } + + +def noop_lru_cache(maxsize=None, typed=False): + """Decorator that serves as a mock of the functools.lru_cache decorator, which + is only available in Python 3. We use this mock as a placeholder in Python 2 + to avoid blowing up when the real decorator isn't available. It merely + calls through to the decorated function and provides a dummy cache_info() + function. + """ + def cache_info(): + return 'No cache info available. Cache is disabled.' + + def cache_clear(): + pass + + def real_noop_lru_cache(fn): + def wrapper(*args, **kwargs): + return fn(*args, **kwargs) + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return wrapper + return real_noop_lru_cache diff --git a/lira/test/data/config.json b/lira/test/data/config.json index ac72fe31..76eff141 100644 --- a/lira/test/data/config.json +++ b/lira/test/data/config.json @@ -11,7 +11,7 @@ "subscription_id": "3e3e176b-629f-46ea-b01d-e36bd650dc54", "analysis_wdls": ["https://raw.githubusercontent.com/HumanCellAtlas/skylab/10x_v0.1.0/10x/count/count.wdl"], "options_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/10x_v0.1.0/adapter_pipelines/10x/options.json", - "wdl_default_inputs_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/10x_v0.1.0/adapter_pipelines/10x/adapter_example_static.json", + "wdl_static_inputs_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/10x_v0.1.0/adapter_pipelines/10x/adapter_example_static.json", "wdl_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/v0.1.0/adapter_pipelines/10x/adapter.wdl", "workflow_name": "Adapter10xCount" }, @@ -26,7 +26,7 @@ "https://raw.githubusercontent.com/HumanCellAtlas/skylab/smartseq2_v0.2.0/pipelines/tasks/rsem.wdl" ], "options_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/v0.1.5/adapter_pipelines/smart_seq2/options.json", - "wdl_default_inputs_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/v0.1.5/adapter_pipelines/smart_seq2/adapter_example_static_demo.json", + "wdl_static_inputs_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/v0.1.5/adapter_pipelines/smart_seq2/adapter_example_static_demo.json", "wdl_link": "https://raw.githubusercontent.com/HumanCellAtlas/pipeline-tools/v0.1.5/adapter_pipelines/smart_seq2/adapter.wdl", "workflow_name": "AdapterSs2RsemSingleSample" } diff --git a/lira/test/test.sh b/lira/test/test.sh index 3963e9ef..4b2c9c55 100755 --- a/lira/test/test.sh +++ b/lira/test/test.sh @@ -3,4 +3,4 @@ docker build -t lira:test ../.. # Run unit tests in docker container -docker run -e listener_config=config.json --entrypoint python lira:test -m unittest discover -v +docker run -e listener_config=config.json --entrypoint python3 lira:test -m unittest discover -v diff --git a/lira/test/test_config.py b/lira/test/test_config.py index 6e0ba517..9006e23f 100644 --- a/lira/test/test_config.py +++ b/lira/test/test_config.py @@ -2,7 +2,7 @@ import unittest import json from copy import deepcopy -from lira import config +from lira import lira_config import os import sys @@ -15,21 +15,21 @@ def setUpClass(cls): # Change to test directory, as tests may have been invoked from another dir dir = os.path.abspath(os.path.dirname(__file__)) os.chdir(dir) - with open('data/config.json', 'rb') as f: + with open('data/config.json', 'r') as f: cls.correct_test_config = json.load(f) def test_correct_config_throws_no_errors(self): test_config = deepcopy(self.correct_test_config) - try: # make sure ListenerConfig executes - result = config.ListenerConfig(test_config) + try: # make sure LiraConfig executes + result = lira_config.LiraConfig(test_config) except BaseException as exception: self.fail( - 'ListenerConfig constructor raised an exception: {exception}' + 'LiraConfig constructor raised an exception: {exception}' .format(exception=exception)) # check correct type - self.assertIsInstance(result, config.ListenerConfig) + self.assertIsInstance(result, lira_config.LiraConfig) def test_config_missing_required_field_throws_value_error(self): @@ -40,21 +40,32 @@ def delete_wdl_field(field_name): return test_config mangled_config = delete_wdl_field('subscription_id') - self.assertRaises(ValueError, config.ListenerConfig, mangled_config) + self.assertRaises(ValueError, lira_config.LiraConfig, mangled_config) mangled_config = delete_wdl_field('workflow_name') - self.assertRaises(ValueError, config.ListenerConfig, mangled_config) + self.assertRaises(ValueError, lira_config.LiraConfig, mangled_config) mangled_config = delete_wdl_field('wdl_link') - self.assertRaises(ValueError, config.ListenerConfig, mangled_config) - mangled_config = delete_wdl_field('wdl_default_inputs_link') - self.assertRaises(ValueError, config.ListenerConfig, mangled_config) + self.assertRaises(ValueError, lira_config.LiraConfig, mangled_config) + mangled_config = delete_wdl_field('wdl_static_inputs_link') + self.assertRaises(ValueError, lira_config.LiraConfig, mangled_config) mangled_config = delete_wdl_field('analysis_wdls') - self.assertRaises(ValueError, config.ListenerConfig, mangled_config) + self.assertRaises(ValueError, lira_config.LiraConfig, mangled_config) def test_error_thrown_when_analysis_wdl_is_not_list(self): test_config = deepcopy(self.correct_test_config) test_config['wdls'][0]['analysis_wdls'] = 'bare string' with self.assertRaises(TypeError): - config.ListenerConfig(test_config) + lira_config.LiraConfig(test_config) + + def test_cache_wdls_is_true_by_default(self): + test_config = deepcopy(self.correct_test_config) + config = lira_config.LiraConfig(test_config) + self.assertTrue(config.cache_wdls) + + def test_cache_wdls_can_be_set_to_false(self): + test_config = deepcopy(self.correct_test_config) + test_config['cache_wdls'] = False + config = lira_config.LiraConfig(test_config) + self.assertFalse(config.cache_wdls) def test_config_duplicate_wdl_raises_value_error(self): @@ -65,22 +76,22 @@ def add_duplicate_wdl_definition(): return test_config mangled_config = add_duplicate_wdl_definition() - self.assertRaises(ValueError, config.ListenerConfig, mangled_config) + self.assertRaises(ValueError, lira_config.LiraConfig, mangled_config) - def test_listener_config_exposes_all_methods_requested_in_notifications(self): + def test_lira_config_exposes_all_methods_requested_in_notifications(self): # test that the calls made in notifications refer to attributes that - # ListenerConfig exposes - test_config = config.ListenerConfig(self.correct_test_config) - requested_listener_attributes = [ + # LiraConfig exposes + test_config = lira_config.LiraConfig(self.correct_test_config) + requested_lira_attributes = [ 'notification_token', 'wdls', 'cromwell_url', 'cromwell_user', 'cromwell_password', 'MAX_CONTENT_LENGTH'] - for attr in requested_listener_attributes: + for attr in requested_lira_attributes: self.assertTrue(hasattr(test_config, attr), 'missing attribute %s' % attr) # get an example wdl to test wdl = test_config.wdls[0] requested_wdl_attributes = [ - 'subscription_id', 'wdl_link', 'workflow_name', 'wdl_default_inputs_link', + 'subscription_id', 'wdl_link', 'workflow_name', 'wdl_static_inputs_link', 'options_link'] for attr in requested_wdl_attributes: self.assertTrue(hasattr(wdl, attr), 'missing attribute %s' % attr) diff --git a/lira/test/test_lira_utils.py b/lira/test/test_lira_utils.py index 8462e56c..e1f4c120 100644 --- a/lira/test/test_lira_utils.py +++ b/lira/test/test_lira_utils.py @@ -40,7 +40,7 @@ def test_compose_inputs(self): inputs = lira_utils.compose_inputs('foo', 'bar', 'baz', 'asdf') self.assertEqual(inputs['foo.bundle_uuid'], 'bar') self.assertEqual(inputs['foo.bundle_version'], 'baz') - self.assertEquals(inputs['foo.runtime_environment'], 'asdf') + self.assertEqual(inputs['foo.runtime_environment'], 'asdf') if __name__ == '__main__': diff --git a/lira/test/test_notifications.py b/lira/test/test_notifications.py new file mode 100644 index 00000000..0e8524b1 --- /dev/null +++ b/lira/test/test_notifications.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +import unittest +import json +from copy import deepcopy +from lira import lira_config +import os +import sys +import requests_mock +from lira.api.notifications import create_prepare_submission_function +from lira import lira_utils + +try: + from functools import lru_cache + cache_available = True +except ImportError: + cache_available = False + + +class TestNotifications(unittest.TestCase): + + @classmethod + def setUpClass(self): + """load the config file""" + # Change to test directory, as tests may have been invoked from another dir + dir = os.path.abspath(os.path.dirname(__file__)) + os.chdir(dir) + with open('data/config.json', 'r') as f: + raw_config = json.load(f) + self.test_config = lira_config.LiraConfig(raw_config) + self.wdl_configs = self.test_config.wdls + self.tenx_config = self.wdl_configs[0] + self.ss2_config = self.wdl_configs[1] + self.submit_wdl = self.test_config.submit_wdl + + def set_up_mock(self, test_config, mock_request): + mock_request.get(test_config.submit_wdl, text=test_config.submit_wdl) + wdl_configs = test_config.wdls + for wc in wdl_configs: + mock_request.get(wc.wdl_link, text=wc.wdl_link) + mock_request.get(wc.wdl_static_inputs_link, text=wc.wdl_static_inputs_link) + mock_request.get(wc.options_link, text=wc.options_link) + for url in wc.analysis_wdls: + mock_request.get(url, text=url) + + @requests_mock.mock() + def test_first_call_writes_to_cache(self, mock_request): + self.set_up_mock(self.test_config, mock_request) + prepare_submission = create_prepare_submission_function(True) + prepare_submission(self.ss2_config, self.submit_wdl) + info = prepare_submission.cache_info() + if cache_available: + self.assertEqual([info.hits, info.misses, info.currsize], [0, 1, 1]) + else: + self.assertIsNotNone(info) + + @requests_mock.mock() + def test_unique_calls_write_to_cache(self, mock_request): + self.set_up_mock(self.test_config, mock_request) + prepare_submission = create_prepare_submission_function(True) + prepare_submission(self.ss2_config, self.submit_wdl) + prepare_submission(self.tenx_config, self.submit_wdl) + info = prepare_submission.cache_info() + if cache_available: + self.assertEqual([info.hits, info.misses, info.currsize], [0, 2, 2]) + else: + self.assertIsNotNone(info) + + @requests_mock.mock() + def test_repeated_calls_hit_cache(self, mock_request): + self.set_up_mock(self.test_config, mock_request) + prepare_submission = create_prepare_submission_function(True) + prepare_submission(self.ss2_config, self.submit_wdl) + prepare_submission(self.ss2_config, self.submit_wdl) + prepare_submission(self.ss2_config, self.submit_wdl) + info = prepare_submission.cache_info() + if cache_available: + self.assertEqual([info.hits, info.misses, info.currsize], [2, 1, 1]) + else: + self.assertIsNotNone(info) + + @requests_mock.mock() + def test_one_repeat_one_new_touches_cache_correctly(self, mock_request): + self.set_up_mock(self.test_config, mock_request) + prepare_submission = create_prepare_submission_function(True) + prepare_submission(self.ss2_config, self.submit_wdl) + prepare_submission(self.ss2_config, self.submit_wdl) + prepare_submission(self.tenx_config, self.submit_wdl) + info = prepare_submission.cache_info() + if cache_available: + self.assertEqual([info.hits, info.misses, info.currsize], [1, 2, 2]) + else: + self.assertIsNotNone(info) + + @requests_mock.mock() + def test_two_unique_then_repeat_touches_cache_correctly(self, mock_request): + self.set_up_mock(self.test_config, mock_request) + prepare_submission = create_prepare_submission_function(True) + prepare_submission(self.ss2_config, self.submit_wdl) + prepare_submission(self.tenx_config, self.submit_wdl) + prepare_submission(self.ss2_config, self.submit_wdl) + info = prepare_submission.cache_info() + if cache_available: + self.assertEqual([info.hits, info.misses, info.currsize], [1, 2, 2]) + else: + self.assertIsNotNone(info) + + @requests_mock.mock() + def test_prepare_submission_returns_correct_object(self, mock_request): + self.set_up_mock(self.test_config, mock_request) + prepare_submission = create_prepare_submission_function(True) + s1 = prepare_submission(self.ss2_config, self.submit_wdl) + self.assertIn(b'smart_seq2/adapter.wdl', s1.wdl_file) + s2 = prepare_submission(self.tenx_config, self.submit_wdl) + self.assertIn(b'10x/adapter.wdl', s2.wdl_file) + s3 = prepare_submission(self.ss2_config, self.submit_wdl) + self.assertIn(b'smart_seq2/adapter.wdl', s3.wdl_file) + s4 = prepare_submission(self.tenx_config, self.submit_wdl) + self.assertIn(b'10x/adapter.wdl', s4.wdl_file) + + @requests_mock.mock() + def test_prepare_submission_does_not_cache_if_disabled(self, mock_request): + self.set_up_mock(self.test_config, mock_request) + prepare_submission = create_prepare_submission_function(False) + prepare_submission(self.ss2_config, self.submit_wdl) + requests_after_one_call = mock_request.call_count + prepare_submission(self.ss2_config, self.submit_wdl) + requests_after_two_calls = mock_request.call_count + self.assertTrue(requests_after_two_calls > requests_after_one_call) + +if __name__ == "__main__": + unittest.main() diff --git a/lira/test/test_service.py b/lira/test/test_service.py index 910e4ec3..c1b8643c 100644 --- a/lira/test/test_service.py +++ b/lira/test/test_service.py @@ -5,7 +5,7 @@ class TestService(unittest.TestCase): def test_health_check(self): - self.assertEquals(health.get(), dict(status='healthy')) + self.assertEqual(health.get(), dict(status='healthy')) if __name__ == '__main__':