Merge pull request #1004 from openzim/relaxed-nautilus

Relaxed nautilus schema

dispatcher/backend/src/common/constants.py

@@ -117,3 +117,4 @@
 
 # OFFLINERS
 ZIMIT_USE_RELAXED_SCHEMA = bool(os.getenv("ZIMIT_USE_RELAXED_SCHEMA"))
+NAUTILUS_USE_RELAXED_SCHEMA = bool(os.getenv("NAUTILUS_USE_RELAXED_SCHEMA"))

dispatcher/backend/src/common/schemas/models.py

@@ -27,6 +27,7 @@
     KolibriFlagsSchema,
     MWOfflinerFlagsSchema,
     NautilusFlagsSchema,
+    NautilusFlagsSchemaRelaxed,
     OpenedxFlagsSchema,
     PhetFlagsSchema,
     SotokiFlagsSchema,
@@ -84,7 +85,11 @@ def get_offliner_schema(offliner):
             Offliner.gutenberg: GutenbergFlagsSchema,
             Offliner.phet: PhetFlagsSchema,
             Offliner.sotoki: SotokiFlagsSchema,
-            Offliner.nautilus: NautilusFlagsSchema,
+            Offliner.nautilus: (
+                NautilusFlagsSchemaRelaxed
+                if constants.NAUTILUS_USE_RELAXED_SCHEMA
+                else NautilusFlagsSchema
+            ),
             Offliner.ted: TedFlagsSchema,
             Offliner.openedx: OpenedxFlagsSchema,
             Offliner.zimit: (

dispatcher/backend/src/common/schemas/offliners/__init__.py

@@ -4,7 +4,10 @@
 from common.schemas.offliners.ifixit import IFixitFlagsSchema
 from common.schemas.offliners.kolibri import KolibriFlagsSchema
 from common.schemas.offliners.mwoffliner import MWOfflinerFlagsSchema
-from common.schemas.offliners.nautilus import NautilusFlagsSchema
+from common.schemas.offliners.nautilus import (
+    NautilusFlagsSchema,
+    NautilusFlagsSchemaRelaxed,
+)
 from common.schemas.offliners.openedx import OpenedxFlagsSchema
 from common.schemas.offliners.sotoki import SotokiFlagsSchema
 from common.schemas.offliners.ted import TedFlagsSchema
@@ -19,6 +22,7 @@
     "KolibriFlagsSchema",
     "MWOfflinerFlagsSchema",
     "NautilusFlagsSchema",
+    "NautilusFlagsSchemaRelaxed",
     "OpenedxFlagsSchema",
     "SotokiFlagsSchema",
     "TedFlagsSchema",

dispatcher/backend/src/common/schemas/offliners/nautilus.py

@@ -186,3 +186,19 @@ class Meta:
         falsy=[False],
         metadata={"label": "Debug", "description": "Enable verbose output"},
     )
+
+
+class NautilusFlagsSchemaRelaxed(NautilusFlagsSchema):
+    """A Nautils flags schema with relaxed constraints on validation
+
+    For now, only zim_file name is not checked anymore.
+    Typically used for nautilus.kiwix.org
+    """
+
+    zim_file = String(
+        metadata={
+            "label": "ZIM filename",
+            "description": "ZIM file name (based on --name if not provided).",
+        },
+        data_key="zim-file",
+    )

dispatcher/backend/src/common/schemas/offliners/zimit.py

@@ -564,8 +564,7 @@ class ZimitFlagsSchemaRelaxed(ZimitFlagsSchema):
     zim_file = String(
         metadata={
             "label": "ZIM filename",
-            "description": "ZIM file name (based on --name if not provided). "
-            "Make sure to end with _{period}.zim",
+            "description": "ZIM file name (based on --name if not provided).",
         },
         data_key="zim-file",
     )

dispatcher/backend/src/tests/integration/routes/schedules/test_nautilus.py

@@ -0,0 +1,91 @@
+from collections import namedtuple
+from typing import List
+
+import pytest
+from utils_for_tests import update_dict
+
+from common import constants
+
+
+class TestNautilus:
+    mod = namedtuple("Modification", ["key_path", "new_value"])
+
+    @pytest.mark.parametrize(
+        "modifications, relaxed_schema, succeeds",
+        [
+            (
+                [mod(key_path="name", new_value="nautilus_test_good_name_not_relaxed")],
+                False,
+                True,
+            ),
+            (
+                [mod(key_path="name", new_value="nautilus_test_good_name_relaxed")],
+                True,
+                True,
+            ),
+            (
+                [
+                    mod(
+                        key_path="name", new_value="nautilus_test_bad_name_not_relaxed"
+                    ),
+                    mod(key_path="config.flags.zim-file", new_value="bad_name"),
+                ],
+                False,
+                False,
+            ),
+            (
+                [
+                    mod(key_path="name", new_value="nautilus_test_bad_name_relaxed"),
+                    mod(key_path="config.flags.zim-file", new_value="bad_name"),
+                ],
+                True,
+                True,
+            ),
+        ],
+    )
+    def test_create_nautilus_schedule_generic(
+        self,
+        client,
+        access_token,
+        garbage_collector,
+        modifications: List[mod],
+        relaxed_schema: bool,
+        succeeds: bool,
+    ):
+        constants.NAUTILUS_USE_RELAXED_SCHEMA = relaxed_schema
+        schedule = {
+            "name": "nautilus_test_ok",
+            "category": "other",
+            "enabled": False,
+            "tags": [],
+            "language": {"code": "fr", "name_en": "French", "name_native": "Français"},
+            "config": {
+                "task_name": "nautilus",
+                "warehouse_path": "/other",
+                "image": {"name": "openzim/nautilus", "tag": "1.0.0"},
+                "monitor": False,
+                "platform": None,
+                "flags": {
+                    "name": "acme",
+                    "collection": "https://www.acme.com",
+                    "zim-file": "acme_en_all_{period}.zim",
+                },
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
+            },
+            "periodicity": "quarterly",
+        }
+        for modification in modifications:
+            update_dict(schedule, modification.key_path, modification.new_value)
+        url = "/schedules/"
+        response = client.post(
+            url, json=schedule, headers={"Authorization": access_token}
+        )
+        response_data = response.get_json()
+        if "_id" in response_data:
+            garbage_collector.add_schedule_id(response_data["_id"])
+        if succeeds:
+            assert response.status_code == 201
+        else:
+            assert response.status_code == 400
+            assert "error_description" in response_data
+            assert "zim-file" in response_data["error_description"]

dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py

@@ -2,29 +2,11 @@
 from typing import List
 
 import pytest
+from utils_for_tests import update_dict
 
 from common import constants
 
 
-def update_dict(dict: dict, key_path: str, new_value: any):
-    """Update a nested key value in a dictionary
-
-    E.g if key_path is 'key1.subkey2', then dict['key1']['subkey2'] will be set"""
-
-    # Split the key path into individual keys
-    keys = key_path.split(".")
-
-    # Initialize a reference to the nested dictionary
-    current_dict = dict
-
-    # Navigate through the nested structure
-    for key in keys[:-1]:
-        current_dict = current_dict[key]
-
-    # Update the value using the last key
-    current_dict[keys[-1]] = new_value
-
-
 class TestZimit:
     mod = namedtuple("Modification", ["key_path", "new_value"])
 

dispatcher/backend/src/tests/utils_for_tests.py

@@ -28,3 +28,22 @@ def patch_dict(data, patch):
             # If the key is not present in the original dictionary, set it with the
             # patch value
             data[key] = patch_value
+
+
+def update_dict(dict: dict, key_path: str, new_value: any):
+    """Update a nested key value in a dictionary
+
+    E.g if key_path is 'key1.subkey2', then dict['key1']['subkey2'] will be set"""
+
+    # Split the key path into individual keys
+    keys = key_path.split(".")
+
+    # Initialize a reference to the nested dictionary
+    current_dict = dict
+
+    # Navigate through the nested structure
+    for key in keys[:-1]:
+        current_dict = current_dict[key]
+
+    # Update the value using the last key
+    current_dict[keys[-1]] = new_value