From 3b5a8386a01f3c007e9d0743eb134ce9bd5f2df2 Mon Sep 17 00:00:00 2001 From: Jeremy Banker Date: Thu, 13 Apr 2023 18:12:23 +0000 Subject: [PATCH 1/5] Update functional test to run without deployed microservice Signed-off-by: Jeremy Banker --- code/utils.py | 17 ++++++++++++----- .../functional_test.py | 16 ++++++++++++---- .../test_configs/wget.yml | 2 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/code/utils.py b/code/utils.py index d2d63d1..c7a9846 100644 --- a/code/utils.py +++ b/code/utils.py @@ -3,23 +3,30 @@ import functools import signal +import threading +import logging class TimeoutExpiredError(Exception): pass def timeout(seconds: int=1): + logger = logging.getLogger("TimeoutDecoratorLogger") def decorator(func): def _handler(signum, frame): raise TimeoutExpiredError(f'Function {func.__name__} timed out after {seconds} seconds.',func.__name__, seconds) @functools.wraps(func) def wrapper(*args, **kwargs): - signal.signal(signal.SIGALRM, _handler) - signal.alarm(seconds) - try: + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGALRM, _handler) + signal.alarm(seconds) + try: + result = func(*args, **kwargs) + finally: + signal.alarm(0) + else: + logger.warning('Running inside non-main thread. Timeout is not available in this context.') result = func(*args, **kwargs) - finally: - signal.alarm(0) return result return wrapper diff --git a/tests/automated_functional_test/functional_test.py b/tests/automated_functional_test/functional_test.py index a5d9d3a..537cec6 100644 --- a/tests/automated_functional_test/functional_test.py +++ b/tests/automated_functional_test/functional_test.py @@ -1,12 +1,19 @@ # Copyright 2023 VMware, Inc. # SPDX-License-Identifier: BSD-2 +import sys +import os +import threading + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "code")) + from argparse import ArgumentParser import os import sys import logging import urllib.parse -import requests +from fastapi.testclient import TestClient +from microservice import microservice_api from typing import List from pydantic import parse from pydantic_yaml import YamlModel, main @@ -68,6 +75,7 @@ def __init__(self, config) -> None: self.failed_test_count = 0 self.passed_test_count = 0 self.logger = logging.getLogger("FunctionalTestRunner") + self.api_client = TestClient(microservice_api) self.load_test_cases() def load_test_cases(self): @@ -102,9 +110,9 @@ def run_one_test(self, test_config): def send_test_to_API(self, test_config): body = test_config.config.input_data - url = urllib.parse.urljoin(self.config.url, self.API_REPORT_PATH) + url = self.API_REPORT_PATH try: - response = requests.post(url=url,data=body.encode('utf-8')) + response = self.api_client.post(url=url,data=body.encode('utf-8')) if response.status_code == 200: return response.json() else: @@ -155,7 +163,7 @@ def check_test_result(self,test_config,API_result): for dependency in test_config.config.expected_dependencies: found = self.check_dependency(dependency, API_result) if not found: - self.logger.debug(f'Incorrect result for test {test_config.name}. Test Failed!') + self.logger.info(f'Incorrect result for test {test_config.name}. Test Failed!\nExpected: {dependency} \nGot {API_result["dependencies"]}') self.failed_test_count += 1 self.failures.append((test_config,f'Expected dependency does not match results. {dependency}')) success = False diff --git a/tests/automated_functional_test/test_configs/wget.yml b/tests/automated_functional_test/test_configs/wget.yml index 93af4a2..f13f194 100644 --- a/tests/automated_functional_test/test_configs/wget.yml +++ b/tests/automated_functional_test/test_configs/wget.yml @@ -371,6 +371,6 @@ config: version: unknown download_location: https://wordpress.org/latest.zip - type: wget - name: wordpress-install-docker-linux.zip + name: wordpress-install-linux.zip version: unknown download_location: https://wordpress.org/latest.zip From 9aa4abebc074bf9b415453ee0e0349cc84353e67 Mon Sep 17 00:00:00 2001 From: Jeremy Banker Date: Thu, 13 Apr 2023 18:17:03 +0000 Subject: [PATCH 2/5] add functional test action Signed-off-by: Jeremy Banker --- .github/workflows/functional-tests.yml | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/functional-tests.yml diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml new file mode 100644 index 0000000..b40908f --- /dev/null +++ b/.github/workflows/functional-tests.yml @@ -0,0 +1,27 @@ +name: Run Functional tests + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r code/requirements.txt + pip install -r automated_functional_test/test_requirements.txt + - name: Run functional tests + run: | + python functional_test.py + working-directory: ./tests/automated_functional_test \ No newline at end of file From 2bee52c3b4eff3666db9af378a79082c392bb540 Mon Sep 17 00:00:00 2001 From: Jeremy Banker Date: Thu, 13 Apr 2023 18:40:14 +0000 Subject: [PATCH 3/5] fix file location Signed-off-by: Jeremy Banker --- .github/workflows/functional-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index b40908f..12e518b 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -20,7 +20,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r code/requirements.txt - pip install -r automated_functional_test/test_requirements.txt + pip install -r tests/automated_functional_test/test_requirements.txt - name: Run functional tests run: | python functional_test.py From 77895dbf96f4baa6bede9a0ad6bc479ebec03326 Mon Sep 17 00:00:00 2001 From: Jeremy Banker Date: Thu, 13 Apr 2023 18:45:37 +0000 Subject: [PATCH 4/5] add test requirements file Signed-off-by: Jeremy Banker --- .github/workflows/functional-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 12e518b..a6ecf24 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -20,6 +20,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r code/requirements.txt + pip install -r tests/requirements.txt pip install -r tests/automated_functional_test/test_requirements.txt - name: Run functional tests run: | From eafe3dac13a91c52da769da612bbf283b20ff2a1 Mon Sep 17 00:00:00 2001 From: Jeremy Banker Date: Thu, 13 Apr 2023 22:10:16 +0000 Subject: [PATCH 5/5] Add test run timer to ensure API response time remains acceptable Signed-off-by: Jeremy Banker --- tests/automated_functional_test/functional_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/automated_functional_test/functional_test.py b/tests/automated_functional_test/functional_test.py index 537cec6..0cb6b36 100644 --- a/tests/automated_functional_test/functional_test.py +++ b/tests/automated_functional_test/functional_test.py @@ -17,6 +17,7 @@ from typing import List from pydantic import parse from pydantic_yaml import YamlModel, main +import time class FindingConfig(YamlModel): source: str @@ -57,6 +58,8 @@ def parse_arguments(): parser.add_argument("--debug", action="store_true") parser.add_argument("--configs","-c",help="Location of the test configurations directory", default=os.path.dirname(os.path.realpath(__file__)) + "/test_configs/") parser.add_argument("--url","-u",help="Base URL for API testing",default="http://localhost:8080/") + parser.add_argument("--max-time", "-t", help="Time in ms to use as a max time a single response is allowed to take", type=int, default=1000) + config = parser.parse_args() return config @@ -104,8 +107,14 @@ def run_all_tests(self): def run_one_test(self, test_config): self.logger.debug(f'Starting test {test_config.name}') + start_time = time.monotonic() API_result = self.send_test_to_API(test_config) + time_taken = time.monotonic() - start_time success = self.check_test_result(test_config, API_result) + if time_taken > self.config.max_time/1000: + success = False + self.failed_test_count += 1 + self.failures.append((test_config,f'Test {test_config.name} took {int(time_taken*1000)}ms. Exceeded max of {self.config.max_time}ms.')) return success def send_test_to_API(self, test_config):