From ab3c062bf7585205d1b5417ec12a07ccf875a500 Mon Sep 17 00:00:00 2001 From: lbarber Date: Thu, 30 Nov 2023 14:37:50 -0500 Subject: [PATCH] test: add experiment integration tests that replace the old unit tests Continue migration towards a behavior-driven style of testing for the REST endpoints and away from heavy mocking. Closes #310 --- tests/unit/restapi/experiment/__init__.py | 16 - .../restapi/experiment/test_controller.py | 163 ------- .../unit/restapi/experiment/test_interface.py | 68 --- tests/unit/restapi/experiment/test_model.py | 52 --- tests/unit/restapi/experiment/test_schema.py | 109 ----- tests/unit/restapi/experiment/test_service.py | 188 -------- tests/unit/restapi/test_experiment.py | 442 ++++++++++++++++++ 7 files changed, 442 insertions(+), 596 deletions(-) delete mode 100644 tests/unit/restapi/experiment/__init__.py delete mode 100644 tests/unit/restapi/experiment/test_controller.py delete mode 100644 tests/unit/restapi/experiment/test_interface.py delete mode 100644 tests/unit/restapi/experiment/test_model.py delete mode 100644 tests/unit/restapi/experiment/test_schema.py delete mode 100644 tests/unit/restapi/experiment/test_service.py create mode 100644 tests/unit/restapi/test_experiment.py diff --git a/tests/unit/restapi/experiment/__init__.py b/tests/unit/restapi/experiment/__init__.py deleted file mode 100644 index ab0a41a34..000000000 --- a/tests/unit/restapi/experiment/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# This Software (Dioptra) is being made available as a public service by the -# National Institute of Standards and Technology (NIST), an Agency of the United -# States Department of Commerce. This software was developed in part by employees of -# NIST and in part by NIST contractors. Copyright in portions of this software that -# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant -# to Title 17 United States Code Section 105, works of NIST employees are not -# subject to copyright protection in the United States. However, NIST may hold -# international copyright in software created by its employees and domestic -# copyright (or licensing rights) in portions of software that were assigned or -# licensed to NIST. To the extent that NIST holds copyright in this software, it is -# being made available under the Creative Commons Attribution 4.0 International -# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts -# of the software developed or licensed by NIST. -# -# ACCESS THE FULL CC BY 4.0 LICENSE HERE: -# https://creativecommons.org/licenses/by/4.0/legalcode diff --git a/tests/unit/restapi/experiment/test_controller.py b/tests/unit/restapi/experiment/test_controller.py deleted file mode 100644 index 5cd25d0ff..000000000 --- a/tests/unit/restapi/experiment/test_controller.py +++ /dev/null @@ -1,163 +0,0 @@ -# This Software (Dioptra) is being made available as a public service by the -# National Institute of Standards and Technology (NIST), an Agency of the United -# States Department of Commerce. This software was developed in part by employees of -# NIST and in part by NIST contractors. Copyright in portions of this software that -# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant -# to Title 17 United States Code Section 105, works of NIST employees are not -# subject to copyright protection in the United States. However, NIST may hold -# international copyright in software created by its employees and domestic -# copyright (or licensing rights) in portions of software that were assigned or -# licensed to NIST. To the extent that NIST holds copyright in this software, it is -# being made available under the Creative Commons Attribution 4.0 International -# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts -# of the software developed or licensed by NIST. -# -# ACCESS THE FULL CC BY 4.0 LICENSE HERE: -# https://creativecommons.org/licenses/by/4.0/legalcode -from __future__ import annotations - -import datetime -from typing import Any, Dict, List - -import pytest -import structlog -from _pytest.monkeypatch import MonkeyPatch -from flask import Flask -from flask_sqlalchemy import SQLAlchemy -from freezegun import freeze_time -from structlog.stdlib import BoundLogger - -from dioptra.restapi.experiment.routes import BASE_ROUTE as EXPERIMENT_BASE_ROUTE -from dioptra.restapi.experiment.service import ExperimentService -from dioptra.restapi.models import Experiment - -LOGGER: BoundLogger = structlog.stdlib.get_logger() - - -@pytest.fixture -def experiment_registration_request() -> Dict[str, Any]: - return {"name": "mnist"} - - -def test_experiment_resource_get(app: Flask, monkeypatch: MonkeyPatch) -> None: - def mockgetall(self, *args, **kwargs) -> List[Experiment]: - LOGGER.info("Mocking ExperimentService.get_all()") - experiment: Experiment = Experiment( - experiment_id=1, - created_on=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - last_modified=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - name="mnist", - ) - return [experiment] - - monkeypatch.setattr(ExperimentService, "get_all", mockgetall) - - with app.test_client() as client: - response: List[Dict[str, Any]] = client.get( - f"/api/{EXPERIMENT_BASE_ROUTE}/" - ).get_json() - - expected: List[Dict[str, Any]] = [ - { - "experimentId": 1, - "createdOn": "2020-08-17T18:46:28.717559", - "lastModified": "2020-08-17T18:46:28.717559", - "name": "mnist", - } - ] - - assert response == expected - - -@freeze_time("2020-08-17T18:46:28.717559") -def test_experiment_resource_post( - app: Flask, - db: SQLAlchemy, - experiment_registration_request: Dict[str, Any], - monkeypatch: MonkeyPatch, -) -> None: - def mockcreate(*args, **kwargs) -> Experiment: - LOGGER.info("Mocking ExperimentService.create()") - timestamp = datetime.datetime.now() - return Experiment( - experiment_id=1, - created_on=timestamp, - last_modified=timestamp, - name="mnist", - ) - - monkeypatch.setattr(ExperimentService, "create", mockcreate) - - with app.test_client() as client: - response: Dict[str, Any] = client.post( - f"/api/{EXPERIMENT_BASE_ROUTE}/", - content_type="multipart/form-data", - data=experiment_registration_request, - follow_redirects=True, - ).get_json() - LOGGER.info("Response received", response=response) - - expected: Dict[str, Any] = { - "experimentId": 1, - "createdOn": "2020-08-17T18:46:28.717559", - "lastModified": "2020-08-17T18:46:28.717559", - "name": "mnist", - } - - assert response == expected - - -def test_experiment_id_resource_get(app: Flask, monkeypatch: MonkeyPatch) -> None: - def mockgetbyid(self, experiment_id: str, *args, **kwargs) -> Experiment: - LOGGER.info("Mocking ExperimentService.get_by_id()") - return Experiment( - experiment_id=experiment_id, - created_on=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - last_modified=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - name="mnist", - ) - - monkeypatch.setattr(ExperimentService, "get_by_id", mockgetbyid) - experiment_id: int = 1 - - with app.test_client() as client: - response: Dict[str, Any] = client.get( - f"/api/{EXPERIMENT_BASE_ROUTE}/{experiment_id}" - ).get_json() - - expected: Dict[str, Any] = { - "experimentId": 1, - "createdOn": "2020-08-17T18:46:28.717559", - "lastModified": "2020-08-17T18:46:28.717559", - "name": "mnist", - } - - assert response == expected - - -def test_experiment_name_resource_get(app: Flask, monkeypatch: MonkeyPatch) -> None: - def mockgetbyname(self, experiment_name: str, *args, **kwargs) -> Experiment: - LOGGER.info("Mocking ExperimentService.get_by_name()") - return Experiment( - experiment_id=1, - created_on=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - last_modified=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - name=experiment_name, - ) - - monkeypatch.setattr(ExperimentService, "get_by_name", mockgetbyname) - experiment_name: str = "mnist" - - with app.test_client() as client: - response: Dict[str, Any] = client.get( - f"/api/{EXPERIMENT_BASE_ROUTE}/name/{experiment_name}" - ).get_json() - - expected: Dict[str, Any] = { - "experimentId": 1, - "createdOn": "2020-08-17T18:46:28.717559", - "lastModified": "2020-08-17T18:46:28.717559", - "name": "mnist", - } - - assert response == expected diff --git a/tests/unit/restapi/experiment/test_interface.py b/tests/unit/restapi/experiment/test_interface.py deleted file mode 100644 index e9a1e99ed..000000000 --- a/tests/unit/restapi/experiment/test_interface.py +++ /dev/null @@ -1,68 +0,0 @@ -# This Software (Dioptra) is being made available as a public service by the -# National Institute of Standards and Technology (NIST), an Agency of the United -# States Department of Commerce. This software was developed in part by employees of -# NIST and in part by NIST contractors. Copyright in portions of this software that -# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant -# to Title 17 United States Code Section 105, works of NIST employees are not -# subject to copyright protection in the United States. However, NIST may hold -# international copyright in software created by its employees and domestic -# copyright (or licensing rights) in portions of software that were assigned or -# licensed to NIST. To the extent that NIST holds copyright in this software, it is -# being made available under the Creative Commons Attribution 4.0 International -# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts -# of the software developed or licensed by NIST. -# -# ACCESS THE FULL CC BY 4.0 LICENSE HERE: -# https://creativecommons.org/licenses/by/4.0/legalcode -from __future__ import annotations - -import datetime - -import pytest -import structlog -from structlog.stdlib import BoundLogger - -from dioptra.restapi.experiment.interface import ( - ExperimentInterface, - ExperimentUpdateInterface, -) -from dioptra.restapi.models import Experiment - -LOGGER: BoundLogger = structlog.stdlib.get_logger() - - -@pytest.fixture -def experiment_interface() -> ExperimentInterface: - return ExperimentInterface( - experiment_id=1, - created_on=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - last_modified=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - name="mnist", - ) - - -@pytest.fixture -def experiment_update_interface() -> ExperimentUpdateInterface: - return ExperimentUpdateInterface(name="group-mnist") - - -def test_ExperimentInterface_create(experiment_interface: ExperimentInterface) -> None: - assert isinstance(experiment_interface, dict) - - -def test_ExperimentUpdateInterface_create( - experiment_update_interface: ExperimentUpdateInterface, -) -> None: - assert isinstance(experiment_update_interface, dict) - - -def test_ExperimentInterface_works(experiment_interface: ExperimentInterface) -> None: - experiment: Experiment = Experiment(**experiment_interface) - assert isinstance(experiment, Experiment) - - -def test_ExperimentUpdateInterface_works( - experiment_update_interface: ExperimentUpdateInterface, -) -> None: - experiment: Experiment = Experiment(**experiment_update_interface) - assert isinstance(experiment, Experiment) diff --git a/tests/unit/restapi/experiment/test_model.py b/tests/unit/restapi/experiment/test_model.py deleted file mode 100644 index 8534b411f..000000000 --- a/tests/unit/restapi/experiment/test_model.py +++ /dev/null @@ -1,52 +0,0 @@ -# This Software (Dioptra) is being made available as a public service by the -# National Institute of Standards and Technology (NIST), an Agency of the United -# States Department of Commerce. This software was developed in part by employees of -# NIST and in part by NIST contractors. Copyright in portions of this software that -# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant -# to Title 17 United States Code Section 105, works of NIST employees are not -# subject to copyright protection in the United States. However, NIST may hold -# international copyright in software created by its employees and domestic -# copyright (or licensing rights) in portions of software that were assigned or -# licensed to NIST. To the extent that NIST holds copyright in this software, it is -# being made available under the Creative Commons Attribution 4.0 International -# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts -# of the software developed or licensed by NIST. -# -# ACCESS THE FULL CC BY 4.0 LICENSE HERE: -# https://creativecommons.org/licenses/by/4.0/legalcode -from __future__ import annotations - -import datetime - -import pytest -import structlog -from structlog.stdlib import BoundLogger - -from dioptra.restapi.models import Experiment, ExperimentRegistrationFormData - -LOGGER: BoundLogger = structlog.stdlib.get_logger() - - -@pytest.fixture -def experiment() -> Experiment: - return Experiment( - experiment_id=1, - created_on=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - last_modified=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - name="mnist", - ) - - -@pytest.fixture -def experiment_registration_form_data() -> ExperimentRegistrationFormData: - return ExperimentRegistrationFormData(name="mnist") - - -def test_Experiment_create(experiment: Experiment) -> None: - assert isinstance(experiment, Experiment) - - -def test_ExperimentRegistrationFormData_create( - experiment_registration_form_data: ExperimentRegistrationFormData, -) -> None: - assert isinstance(experiment_registration_form_data, dict) diff --git a/tests/unit/restapi/experiment/test_schema.py b/tests/unit/restapi/experiment/test_schema.py deleted file mode 100644 index 3c87d9aee..000000000 --- a/tests/unit/restapi/experiment/test_schema.py +++ /dev/null @@ -1,109 +0,0 @@ -# This Software (Dioptra) is being made available as a public service by the -# National Institute of Standards and Technology (NIST), an Agency of the United -# States Department of Commerce. This software was developed in part by employees of -# NIST and in part by NIST contractors. Copyright in portions of this software that -# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant -# to Title 17 United States Code Section 105, works of NIST employees are not -# subject to copyright protection in the United States. However, NIST may hold -# international copyright in software created by its employees and domestic -# copyright (or licensing rights) in portions of software that were assigned or -# licensed to NIST. To the extent that NIST holds copyright in this software, it is -# being made available under the Creative Commons Attribution 4.0 International -# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts -# of the software developed or licensed by NIST. -# -# ACCESS THE FULL CC BY 4.0 LICENSE HERE: -# https://creativecommons.org/licenses/by/4.0/legalcode -from __future__ import annotations - -import datetime -from typing import Any, Dict - -import pytest -import structlog -from flask import Flask -from structlog.stdlib import BoundLogger - -from dioptra.restapi.experiment.schema import ( - ExperimentRegistrationFormSchema, - ExperimentSchema, -) -from dioptra.restapi.models import Experiment, ExperimentRegistrationForm - -LOGGER: BoundLogger = structlog.stdlib.get_logger() - - -@pytest.fixture -def experiment_registration_form(app: Flask) -> ExperimentRegistrationForm: - with app.test_request_context(): - form = ExperimentRegistrationForm(data={"name": "mnist"}) - - return form - - -@pytest.fixture -def experiment_schema() -> ExperimentSchema: - return ExperimentSchema() - - -@pytest.fixture -def experiment_registration_form_schema() -> ExperimentRegistrationFormSchema: - return ExperimentRegistrationFormSchema() - - -def test_ExperimentSchema_create(experiment_schema: ExperimentSchema) -> None: - assert isinstance(experiment_schema, ExperimentSchema) - - -def test_ExperimentRegistrationFormSchema_create( - experiment_registration_form_schema: ExperimentRegistrationFormSchema, -) -> None: - assert isinstance( - experiment_registration_form_schema, ExperimentRegistrationFormSchema - ) - - -def test_ExperimentSchema_load_works(experiment_schema: ExperimentSchema) -> None: - experiment: Experiment = experiment_schema.load( - { - "experimentId": 1, - "createdOn": "2020-08-17T18:46:28.717559", - "lastModified": "2020-08-17T18:46:28.717559", - "name": "mnist", - } - ) - - assert experiment["experiment_id"] == 1 - assert experiment["created_on"] == datetime.datetime( - 2020, 8, 17, 18, 46, 28, 717559 - ) - assert experiment["last_modified"] == datetime.datetime( - 2020, 8, 17, 18, 46, 28, 717559 - ) - assert experiment["name"] == "mnist" - - -def test_ExperimentSchema_dump_works(experiment_schema: ExperimentSchema) -> None: - experiment: Experiment = Experiment( - experiment_id=1, - created_on=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - last_modified=datetime.datetime(2020, 8, 17, 18, 46, 28, 717559), - name="mnist", - ) - experiment_serialized: Dict[str, Any] = experiment_schema.dump(experiment) - - assert experiment_serialized["experimentId"] == 1 - assert experiment_serialized["createdOn"] == "2020-08-17T18:46:28.717559" - assert experiment_serialized["lastModified"] == "2020-08-17T18:46:28.717559" - assert experiment_serialized["name"] == "mnist" - - -def test_ExperimentRegistrationFormSchema_dump_works( - experiment_registration_form: ExperimentRegistrationForm, - experiment_registration_form_schema: ExperimentRegistrationFormSchema, -) -> None: - experiment_serialized: Dict[str, Any] = experiment_registration_form_schema.dump( - experiment_registration_form - ) - - assert experiment_serialized["name"] == "mnist" diff --git a/tests/unit/restapi/experiment/test_service.py b/tests/unit/restapi/experiment/test_service.py deleted file mode 100644 index 9e35e43ec..000000000 --- a/tests/unit/restapi/experiment/test_service.py +++ /dev/null @@ -1,188 +0,0 @@ -# This Software (Dioptra) is being made available as a public service by the -# National Institute of Standards and Technology (NIST), an Agency of the United -# States Department of Commerce. This software was developed in part by employees of -# NIST and in part by NIST contractors. Copyright in portions of this software that -# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant -# to Title 17 United States Code Section 105, works of NIST employees are not -# subject to copyright protection in the United States. However, NIST may hold -# international copyright in software created by its employees and domestic -# copyright (or licensing rights) in portions of software that were assigned or -# licensed to NIST. To the extent that NIST holds copyright in this software, it is -# being made available under the Creative Commons Attribution 4.0 International -# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts -# of the software developed or licensed by NIST. -# -# ACCESS THE FULL CC BY 4.0 LICENSE HERE: -# https://creativecommons.org/licenses/by/4.0/legalcode -from __future__ import annotations - -import datetime -from typing import List - -import pytest -import structlog -from flask import Flask -from flask_sqlalchemy import SQLAlchemy -from freezegun import freeze_time -from structlog.stdlib import BoundLogger - -from dioptra.restapi.experiment.errors import ExperimentAlreadyExistsError -from dioptra.restapi.experiment.service import ExperimentService -from dioptra.restapi.models import ( - Experiment, - ExperimentRegistrationForm, - ExperimentRegistrationFormData, -) - -LOGGER: BoundLogger = structlog.stdlib.get_logger() - - -@pytest.fixture -def experiment_registration_form(app: Flask) -> ExperimentRegistrationForm: - with app.test_request_context(): - form = ExperimentRegistrationForm(data={"name": "mnist"}) - - return form - - -@pytest.fixture -def experiment_registration_form_data(app: Flask) -> ExperimentRegistrationFormData: - return ExperimentRegistrationFormData(name="mnist") - - -@pytest.fixture -def experiment_service(dependency_injector) -> ExperimentService: - return dependency_injector.get(ExperimentService) - - -@freeze_time("2020-08-17T18:46:28.717559") -def test_create( - db: SQLAlchemy, - experiment_service: ExperimentService, - experiment_registration_form_data: ExperimentRegistrationFormData, - monkeypatch, -): - def mockcreatemlflowexperiment(self, experiment_name: str, *args, **kwargs) -> int: - LOGGER.info( - "Mocking ExperimentService.create_mlflow_experiment()", - experiment_name=experiment_name, - args=args, - kwargs=kwargs, - ) - return 1 - - monkeypatch.setattr( - ExperimentService, "create_mlflow_experiment", mockcreatemlflowexperiment - ) - experiment: Experiment = experiment_service.create( - experiment_registration_form_data=experiment_registration_form_data - ) - - assert experiment.experiment_id == 1 - assert experiment.name == "mnist" - assert experiment.created_on == datetime.datetime(2020, 8, 17, 18, 46, 28, 717559) - assert experiment.last_modified == datetime.datetime( - 2020, 8, 17, 18, 46, 28, 717559 - ) - - with pytest.raises(ExperimentAlreadyExistsError): - experiment_service.create( - experiment_registration_form_data=experiment_registration_form_data - ) - - -@freeze_time("2020-08-17T18:46:28.717559") -def test_get_by_id(db: SQLAlchemy, experiment_service: ExperimentService): - timestamp: datetime.datetime = datetime.datetime.now() - - new_experiment: Experiment = Experiment( - name="mnist", created_on=timestamp, last_modified=timestamp - ) - - db.session.add(new_experiment) - db.session.commit() - - experiment: Experiment = experiment_service.get_by_id(1) - - assert experiment == new_experiment - - -@freeze_time("2020-08-17T18:46:28.717559") -def test_get_by_name(db: SQLAlchemy, experiment_service: ExperimentService): - timestamp: datetime.datetime = datetime.datetime.now() - - new_experiment: Experiment = Experiment( - name="mnist", created_on=timestamp, last_modified=timestamp - ) - - db.session.add(new_experiment) - db.session.commit() - - experiment: Experiment = experiment_service.get_by_name("mnist") - - assert experiment == new_experiment - - -@freeze_time("2020-08-17T18:46:28.717559") -def test_get_all(db: SQLAlchemy, experiment_service: ExperimentService): - timestamp: datetime.datetime = datetime.datetime.now() - - new_experiment1: Experiment = Experiment( - name="mnist", created_on=timestamp, last_modified=timestamp - ) - new_experiment2: Experiment = Experiment( - name="imagenet", created_on=timestamp, last_modified=timestamp - ) - - db.session.add(new_experiment1) - db.session.add(new_experiment2) - db.session.commit() - - results: List[Experiment] = experiment_service.get_all() - - assert len(results) == 2 - assert new_experiment1 in results and new_experiment2 in results - assert new_experiment1.experiment_id == 1 - assert new_experiment2.experiment_id == 2 - - -def test_extract_data_from_form( - experiment_service: ExperimentService, - experiment_registration_form: ExperimentRegistrationForm, -): - experiment_registration_form_data: ExperimentRegistrationFormData = ( - experiment_service.extract_data_from_form( - experiment_registration_form=experiment_registration_form - ) - ) - - assert experiment_registration_form_data["name"] == "mnist" - - -@freeze_time("2020-08-17T18:46:28.717559") -@pytest.mark.parametrize( - "slugified_name,requested_name", - [ - ("mnist", "MNIST"), - ("my-experiment", "My Experiment"), - ("abc_my-experiment", "ABc_My Experiment"), - ], -) -def test_get_by_name_slugification( - db: SQLAlchemy, - experiment_service: ExperimentService, - slugified_name: str, - requested_name: str, -): - timestamp: datetime.datetime = datetime.datetime.now() - - new_experiment: Experiment = Experiment( - name=slugified_name, created_on=timestamp, last_modified=timestamp - ) - - db.session.add(new_experiment) - db.session.commit() - - experiment: Experiment = experiment_service.get_by_name(requested_name) - - assert experiment == new_experiment diff --git a/tests/unit/restapi/test_experiment.py b/tests/unit/restapi/test_experiment.py new file mode 100644 index 000000000..bfc288f63 --- /dev/null +++ b/tests/unit/restapi/test_experiment.py @@ -0,0 +1,442 @@ +# This Software (Dioptra) is being made available as a public service by the +# National Institute of Standards and Technology (NIST), an Agency of the United +# States Department of Commerce. This software was developed in part by employees of +# NIST and in part by NIST contractors. Copyright in portions of this software that +# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant +# to Title 17 United States Code Section 105, works of NIST employees are not +# subject to copyright protection in the United States. However, NIST may hold +# international copyright in software created by its employees and domestic +# copyright (or licensing rights) in portions of software that were assigned or +# licensed to NIST. To the extent that NIST holds copyright in this software, it is +# being made available under the Creative Commons Attribution 4.0 International +# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts +# of the software developed or licensed by NIST. +# +# ACCESS THE FULL CC BY 4.0 LICENSE HERE: +# https://creativecommons.org/licenses/by/4.0/legalcode + +"""Test suite for experiment operations. + +This module contains a set of tests that validate the supported CRUD operations and +additional functionalities for the experiment entity. The tests ensure that the +experiments can be registered, retrieved, and deleted as expected through the REST API. +""" +from __future__ import annotations + +from typing import Any + +from flask.testing import FlaskClient +from flask_sqlalchemy import SQLAlchemy +from werkzeug.test import TestResponse + +from dioptra.restapi.experiment.routes import BASE_ROUTE as EXPERIMENT_BASE_ROUTE + +# -- Actions --------------------------------------------------------------------------- + + +def register_experiment(client: FlaskClient, name: str) -> TestResponse: + """Register an experiment using the API. + + Args: + client: The Flask test client. + name: The name to assign to the new experiment. + + Returns: + The response from the API. + """ + return client.post( + f"/api/{EXPERIMENT_BASE_ROUTE}/", + json={"name": name}, + follow_redirects=True, + ) + + +def rename_experiment_with_name( + client: FlaskClient, name: str, new_name: str +) -> TestResponse: + """Rename an experiment using the API. + + Args: + client: The Flask test client. + name: The old name of the experimet to rename. + new_name: The new name to assign to the experiment. + + Returns: + The response from the API. + """ + return client.put( + f"/api/{EXPERIMENT_BASE_ROUTE}/name/{name}", + json={"name": new_name}, + follow_redirects=True, + ) + + +def rename_experiment_with_id( + client: FlaskClient, id: int, new_name: str +) -> TestResponse: + """Rename an experiment using the API. + + Args: + client: The Flask test client. + id: The id of the experimet to rename. + new_name: The new name to assign to the experiment. + + Returns: + The response from the API. + """ + return client.put( + f"/api/{EXPERIMENT_BASE_ROUTE}/{id}", + json={"name": new_name}, + follow_redirects=True, + ) + + +def delete_experiment_with_name(client: FlaskClient, name: str) -> TestResponse: + """Delete an experiment using the API. + + Args: + client: The Flask test client. + name: The name of the experiment to delete. + + Returns: + The response from the API. + """ + return client.delete( + f"/api/{EXPERIMENT_BASE_ROUTE}/name/{name}", + follow_redirects=True, + ) + + +def delete_experiment_with_id(client: FlaskClient, id: int) -> TestResponse: + """Delete an experiment using the API. + + Args: + client: The Flask test client. + id: The id of the experiment to delete. + + Returns: + The response from the API. + """ + return client.delete( + f"/api/{EXPERIMENT_BASE_ROUTE}/{id}", + follow_redirects=True, + ) + + +# -- Assertions ------------------------------------------------------------------------ + + +def assert_retrieving_experiment_by_name_works( + client: FlaskClient, name: str, expected: dict[str, Any] +) -> None: + """Assert that retrieving a experiment by name works. + + Args: + client: The Flask test client. + name: The name of the experiment to retrieve. + expected: The expected response from the API. + + Raises: + AssertionError: If the response status code is not 200 or if the API response + does not match the expected response. + """ + response = client.get( + f"/api/{EXPERIMENT_BASE_ROUTE}/name/{name}", follow_redirects=True + ) + assert response.status_code == 200 and response.get_json() == expected + + +def assert_retrieving_experiment_by_id_works( + client: FlaskClient, id: int, expected: dict[str, Any] +) -> None: + """Assert that retrieving an experiment by id works. + + Args: + client: The Flask test client. + id: The id of the experiment to retrieve. + expected: The expected response from the API. + + Raises: + AssertionError: If the response status code is not 200 or if the API response + does not match the expected response. + """ + response = client.get(f"/api/{EXPERIMENT_BASE_ROUTE}/{id}", follow_redirects=True) + assert response.status_code == 200 and response.get_json() == expected + + +def assert_retrieving_all_experiments_works( + client: FlaskClient, expected: list[dict[str, Any]] +) -> None: + """Assert that retrieving all experiments works. + + Args: + client: The Flask test client. + expected: The expected response from the API. + + Raises: + AssertionError: If the response status code is not 200 or if the API response + does not match the expected response. + """ + response = client.get(f"/api/{EXPERIMENT_BASE_ROUTE}", follow_redirects=True) + assert response.status_code == 200 and response.get_json() == expected + + +def assert_experiment_name_matches_expected_name( + client: FlaskClient, id: int, expected_name: str +) -> None: + """Assert that the name of an experiment matches the expected name. + + Args: + client: The Flask test client. + id: The id of the experiment to retrieve. + expected_name: The expected name of the experiment. + + Raises: + AssertionError: If the response status code is not 200 or if the name of the + experiment does not match the expected name. + """ + response = client.get( + f"/api/{EXPERIMENT_BASE_ROUTE}/{id}", + follow_redirects=True, + ) + assert response.status_code == 200 and response.get_json()["name"] == expected_name + + +def assert_registering_existing_experiment_name_fails( + client: FlaskClient, name: str +) -> None: + """Assert that registering an experiment with an existing name fails. + + Args: + client: The Flask test client. + name: The name to assign to the new experiment. + + Raises: + AssertionError: If the response status code is not 400. + """ + response = register_experiment(client, name=name) + assert response.status_code == 400 + + +def assert_experiment_is_not_found(client: FlaskClient, id: int) -> None: + """Assert that an experiment is not found. + + Args: + client: The Flask test client. + id: The id of the experiment to retrieve. + + Raises: + AssertionError: If the response status code is not 404. + """ + response = client.get( + f"/api/{EXPERIMENT_BASE_ROUTE}/{id}", + follow_redirects=True, + ) + assert response.status_code == 404 + + +def assert_experiment_count_matches_expected_count( + client: FlaskClient, expected: int +) -> None: + """Assert that the number of experiments matches the expected number. + + Args: + client: The Flask test client. + expected: The expected number of experiments. + + Raises: + AssertionError: If the response status code is not 200 or if the number of + experiments does not match the expected number. + """ + response = client.get( + f"/api/{EXPERIMENT_BASE_ROUTE}", + follow_redirects=True, + ) + assert len(response.get_json()) == expected + + +# -- Tests ----------------------------------------------------------------------------- + + +def test_register_experiment(client: FlaskClient, db: SQLAlchemy) -> None: + """Tests that experiments can be registered following the scenario below:: + + Scenario: Registering an Experiment + Given I am an authorized user, + I need to be able to submit a register request, + in order to register an experiment. + + This test validates by following these actions: + + - A user registers two experiments, "mnist_name" and "mnist_id". + - The user is able to retrieve information about each experiment using the + experiment id or the unique experiment name. + - In both cases, the returned information matches the information that was provided + during registration. + """ + experiment1_expected = register_experiment(client, name="mnist_name").get_json() + experiment2_expected = register_experiment(client, name="mnist_id").get_json() + assert_retrieving_experiment_by_name_works( + client, name=experiment1_expected["name"], expected=experiment1_expected + ) + assert_retrieving_experiment_by_id_works( + client, id=experiment1_expected["experimentId"], expected=experiment1_expected + ) + assert_retrieving_experiment_by_name_works( + client, name=experiment2_expected["name"], expected=experiment2_expected + ) + assert_retrieving_experiment_by_id_works( + client, id=experiment2_expected["experimentId"], expected=experiment2_expected + ) + + +def test_cannot_register_existing_experiment_name( + client: FlaskClient, db: SQLAlchemy +) -> None: + """Test that registering a experiment with an existing name fails. + + This test validates the following sequence of actions: + + - A user registers an experiment named "error". + - The user attempts to register a second experiment with the same name, which fails. + """ + register_experiment(client, name="error") + assert_registering_existing_experiment_name_fails(client, name="error") + + +def test_retrieve_experiment(client: FlaskClient, db: SQLAlchemy) -> None: + """Test that an experiment can be retrieved following the scenario below:: + + Scenario: Get an Experiment + Given I am an authorized user and an experiment exists, + I need to be able to submit a get request + in order to retrieve that experiment using its name and id as identifiers. + + This test validates by following these actions: + + - A user registers an experiment named "retrieve". + - The user is able to retrieve information about the "retrieve" experiment that + matches the information that was provided during registration using its name and + id as identifiers. + - In all cases, the returned information matches the information that was provided + during registration. + """ + experiment_expected = register_experiment(client, name="retrieve").get_json() + assert_retrieving_experiment_by_name_works( + client, name=experiment_expected["name"], expected=experiment_expected + ) + assert_retrieving_experiment_by_id_works( + client, id=experiment_expected["experimentId"], expected=experiment_expected + ) + + +def test_list_experiments(client: FlaskClient, db: SQLAlchemy) -> None: + """Test that the list of experiments can be renamed following the scenario below:: + + Scenario: Get the List of Registered Experiments + Given I am an authorized user and a set of experiments exist, + I need to be able to submit a get request + in order to retrieve the list of registered experiments. + + This test validates by following these actions: + + - A user registers a set of experiments named "mnist1", "mnist2" and "mnist3". + - The user is able to retrieve information about the experiments that + matches the information that was provided during registration. + - The user is able to retrieve a list of all registered experiments. + - The returned list of experiments matches the information that was provided + during registration. + """ + experiment1_expected = register_experiment(client, name="mnist1").get_json() + experiment2_expected = register_experiment(client, name="mnist2").get_json() + experiment3_expected = register_experiment(client, name="mnist3").get_json() + experiment_expected_list = [ + experiment1_expected, + experiment2_expected, + experiment3_expected, + ] + assert_retrieving_all_experiments_works(client, expected=experiment_expected_list) + + +def test_rename_experiment(client: FlaskClient, db: SQLAlchemy) -> None: + """Test that an experiment can be renamed following the scenario below:: + + Scenario: Rename an Experiment + Given I am an authorized user and an experiment exists, + I need to be able to submit a rename request + in order to rename an experiment using its old name and id as identifiers. + + This test validates by following these actions: + + - A user registers an experiment named "mnist". + - The user is able to retrieve information about the "mnist" experiment that + matches the information that was provided during registration. + - The user renames this same experiment to "imagenet" using the experiment's name as + an identifier. + - The user retrieves information about the same experiment and it reflects the name + change. + - The user renames this same experiment again to "fruits360" using the experiment's + id as an identifier. + - The user again retrieves information about the same experiment and it reflects + the name change. + """ + start_name = "mnist" + update1_name = "imagenet" + update2_name = "fruits360" + experiment_json = register_experiment(client, name=start_name).get_json() + assert_experiment_name_matches_expected_name( + client, id=experiment_json["experimentId"], expected_name=start_name + ) + rename_experiment_with_name(client, name="mnist", new_name=update1_name) + assert_experiment_name_matches_expected_name( + client, id=experiment_json["experimentId"], expected_name=update1_name + ) + rename_experiment_with_id( + client, id=experiment_json["experimentId"], new_name=update2_name + ) + assert_experiment_name_matches_expected_name( + client, id=experiment_json["experimentId"], expected_name=update2_name + ) + + +def test_delete_experiment(client: FlaskClient, db: SQLAlchemy) -> None: + """Test that an experiment can be deleted following the scenario below:: + + Scenario: Delete an Experiment + Given I am an authorized user and an experiment exists, + I need to be able to submit a delete request + in order to delete an experiment, using its name and id as identifiers. + + This test validates by following these actions: + + - Deleting by name: + - A user registers an experiment named "delete_name". + - The user is able to retrieve information about the "delete_name" experiment + that matches the information that was provided during registration. + - The user deletes the "delete_name" experiment by referencing its name. + - The user attempts to retrieve information about the "delete_name" experiment, + which is no longer found. + + - Deleting by ID: + - A user registers another experiment named "delete_id". + - The user is able to retrieve information about the "delete_id" experiment that + matches the information that was provided during registration. + - The user deletes the "delete_id" experiment by referencing its id. + - The user attempts to retrieve information about the "delete_id" experiment, + which is no longer found. + """ + name_by_name = "delete_name" + experiment1_json = register_experiment(client, name=name_by_name).get_json() + assert_retrieving_experiment_by_id_works( + client, id=experiment1_json["experimentId"], expected=experiment1_json + ) + delete_experiment_with_name(client, name=name_by_name) + assert_experiment_is_not_found(client, id=experiment1_json["experimentId"]) + + name_by_id = "delete_id" + experiment2_json = register_experiment(client, name=name_by_id).get_json() + assert_retrieving_experiment_by_id_works( + client, id=experiment2_json["experimentId"], expected=experiment2_json + ) + delete_experiment_with_id(client, id=experiment2_json["experimentId"]) + assert_experiment_is_not_found(client, id=experiment2_json["experimentId"])