diff --git a/dispatcher/backend/src/common/constants.py b/dispatcher/backend/src/common/constants.py index c03408d8..e2be4666 100644 --- a/dispatcher/backend/src/common/constants.py +++ b/dispatcher/backend/src/common/constants.py @@ -117,3 +117,4 @@ # OFFLINERS ZIMIT_USE_RELAXED_SCHEMA = bool(os.getenv("ZIMIT_USE_RELAXED_SCHEMA")) +NAUTILUS_USE_RELAXED_SCHEMA = bool(os.getenv("NAUTILUS_USE_RELAXED_SCHEMA")) diff --git a/dispatcher/backend/src/common/schemas/models.py b/dispatcher/backend/src/common/schemas/models.py index d882e35e..e71c885f 100644 --- a/dispatcher/backend/src/common/schemas/models.py +++ b/dispatcher/backend/src/common/schemas/models.py @@ -27,6 +27,7 @@ KolibriFlagsSchema, MWOfflinerFlagsSchema, NautilusFlagsSchema, + NautilusFlagsSchemaRelaxed, OpenedxFlagsSchema, PhetFlagsSchema, SotokiFlagsSchema, @@ -84,7 +85,11 @@ def get_offliner_schema(offliner): Offliner.gutenberg: GutenbergFlagsSchema, Offliner.phet: PhetFlagsSchema, Offliner.sotoki: SotokiFlagsSchema, - Offliner.nautilus: NautilusFlagsSchema, + Offliner.nautilus: ( + NautilusFlagsSchemaRelaxed + if constants.NAUTILUS_USE_RELAXED_SCHEMA + else NautilusFlagsSchema + ), Offliner.ted: TedFlagsSchema, Offliner.openedx: OpenedxFlagsSchema, Offliner.zimit: ( diff --git a/dispatcher/backend/src/common/schemas/offliners/__init__.py b/dispatcher/backend/src/common/schemas/offliners/__init__.py index 252b1e70..f7eebfbe 100644 --- a/dispatcher/backend/src/common/schemas/offliners/__init__.py +++ b/dispatcher/backend/src/common/schemas/offliners/__init__.py @@ -4,7 +4,10 @@ from common.schemas.offliners.ifixit import IFixitFlagsSchema from common.schemas.offliners.kolibri import KolibriFlagsSchema from common.schemas.offliners.mwoffliner import MWOfflinerFlagsSchema -from common.schemas.offliners.nautilus import NautilusFlagsSchema +from common.schemas.offliners.nautilus import ( + NautilusFlagsSchema, + NautilusFlagsSchemaRelaxed, +) from common.schemas.offliners.openedx import OpenedxFlagsSchema from common.schemas.offliners.sotoki import SotokiFlagsSchema from common.schemas.offliners.ted import TedFlagsSchema @@ -19,6 +22,7 @@ "KolibriFlagsSchema", "MWOfflinerFlagsSchema", "NautilusFlagsSchema", + "NautilusFlagsSchemaRelaxed", "OpenedxFlagsSchema", "SotokiFlagsSchema", "TedFlagsSchema", diff --git a/dispatcher/backend/src/common/schemas/offliners/nautilus.py b/dispatcher/backend/src/common/schemas/offliners/nautilus.py index c0d526ed..19e24220 100644 --- a/dispatcher/backend/src/common/schemas/offliners/nautilus.py +++ b/dispatcher/backend/src/common/schemas/offliners/nautilus.py @@ -186,3 +186,19 @@ class Meta: falsy=[False], metadata={"label": "Debug", "description": "Enable verbose output"}, ) + + +class NautilusFlagsSchemaRelaxed(NautilusFlagsSchema): + """A Nautils flags schema with relaxed constraints on validation + + For now, only zim_file name is not checked anymore. + Typically used for nautilus.kiwix.org + """ + + zim_file = String( + metadata={ + "label": "ZIM filename", + "description": "ZIM file name (based on --name if not provided).", + }, + data_key="zim-file", + ) diff --git a/dispatcher/backend/src/common/schemas/offliners/zimit.py b/dispatcher/backend/src/common/schemas/offliners/zimit.py index f4cf38fa..f9427993 100644 --- a/dispatcher/backend/src/common/schemas/offliners/zimit.py +++ b/dispatcher/backend/src/common/schemas/offliners/zimit.py @@ -564,8 +564,7 @@ class ZimitFlagsSchemaRelaxed(ZimitFlagsSchema): zim_file = String( metadata={ "label": "ZIM filename", - "description": "ZIM file name (based on --name if not provided). " - "Make sure to end with _{period}.zim", + "description": "ZIM file name (based on --name if not provided).", }, data_key="zim-file", ) diff --git a/dispatcher/backend/src/tests/integration/routes/schedules/test_nautilus.py b/dispatcher/backend/src/tests/integration/routes/schedules/test_nautilus.py new file mode 100644 index 00000000..ba860946 --- /dev/null +++ b/dispatcher/backend/src/tests/integration/routes/schedules/test_nautilus.py @@ -0,0 +1,91 @@ +from collections import namedtuple +from typing import List + +import pytest +from utils_for_tests import update_dict + +from common import constants + + +class TestNautilus: + mod = namedtuple("Modification", ["key_path", "new_value"]) + + @pytest.mark.parametrize( + "modifications, relaxed_schema, succeeds", + [ + ( + [mod(key_path="name", new_value="nautilus_test_good_name_not_relaxed")], + False, + True, + ), + ( + [mod(key_path="name", new_value="nautilus_test_good_name_relaxed")], + True, + True, + ), + ( + [ + mod( + key_path="name", new_value="nautilus_test_bad_name_not_relaxed" + ), + mod(key_path="config.flags.zim-file", new_value="bad_name"), + ], + False, + False, + ), + ( + [ + mod(key_path="name", new_value="nautilus_test_bad_name_relaxed"), + mod(key_path="config.flags.zim-file", new_value="bad_name"), + ], + True, + True, + ), + ], + ) + def test_create_nautilus_schedule_generic( + self, + client, + access_token, + garbage_collector, + modifications: List[mod], + relaxed_schema: bool, + succeeds: bool, + ): + constants.NAUTILUS_USE_RELAXED_SCHEMA = relaxed_schema + schedule = { + "name": "nautilus_test_ok", + "category": "other", + "enabled": False, + "tags": [], + "language": {"code": "fr", "name_en": "French", "name_native": "Français"}, + "config": { + "task_name": "nautilus", + "warehouse_path": "/other", + "image": {"name": "openzim/nautilus", "tag": "1.0.0"}, + "monitor": False, + "platform": None, + "flags": { + "name": "acme", + "collection": "https://www.acme.com", + "zim-file": "acme_en_all_{period}.zim", + }, + "resources": {"cpu": 3, "memory": 1024, "disk": 0}, + }, + "periodicity": "quarterly", + } + for modification in modifications: + update_dict(schedule, modification.key_path, modification.new_value) + url = "/schedules/" + response = client.post( + url, json=schedule, headers={"Authorization": access_token} + ) + response_data = response.get_json() + if "_id" in response_data: + garbage_collector.add_schedule_id(response_data["_id"]) + if succeeds: + assert response.status_code == 201 + else: + assert response.status_code == 400 + assert "error_description" in response_data + assert "zim-file" in response_data["error_description"] diff --git a/dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py b/dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py index a705cdbc..951f2334 100644 --- a/dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py +++ b/dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py @@ -2,29 +2,11 @@ from typing import List import pytest +from utils_for_tests import update_dict from common import constants -def update_dict(dict: dict, key_path: str, new_value: any): - """Update a nested key value in a dictionary - - E.g if key_path is 'key1.subkey2', then dict['key1']['subkey2'] will be set""" - - # Split the key path into individual keys - keys = key_path.split(".") - - # Initialize a reference to the nested dictionary - current_dict = dict - - # Navigate through the nested structure - for key in keys[:-1]: - current_dict = current_dict[key] - - # Update the value using the last key - current_dict[keys[-1]] = new_value - - class TestZimit: mod = namedtuple("Modification", ["key_path", "new_value"]) diff --git a/dispatcher/backend/src/tests/utils_for_tests.py b/dispatcher/backend/src/tests/utils_for_tests.py index 0c259341..51a09546 100644 --- a/dispatcher/backend/src/tests/utils_for_tests.py +++ b/dispatcher/backend/src/tests/utils_for_tests.py @@ -28,3 +28,22 @@ def patch_dict(data, patch): # If the key is not present in the original dictionary, set it with the # patch value data[key] = patch_value + + +def update_dict(dict: dict, key_path: str, new_value: any): + """Update a nested key value in a dictionary + + E.g if key_path is 'key1.subkey2', then dict['key1']['subkey2'] will be set""" + + # Split the key path into individual keys + keys = key_path.split(".") + + # Initialize a reference to the nested dictionary + current_dict = dict + + # Navigate through the nested structure + for key in keys[:-1]: + current_dict = current_dict[key] + + # Update the value using the last key + current_dict[keys[-1]] = new_value