Merge pull request #80 from openzim/metadata_fixes

Metadata fixes and enhancements

CHANGELOG.md

@@ -8,9 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Language metadata can be customized (#77)
 - New html option in coverage report
 
 ### Fixed
+- Name metadadata is not set correctly (#76)
+- Default publisher is not correctly spelled (#78)
+- Adapt to hatchling v1.19.0 which mandates packages setting (#79)
 - Small fixes in invoke tasks
 
 ### Changed

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "kolibri2zim"
 authors = [{ name = "Kiwix", email = "[email protected]" }]
 keywords = ["kiwix", "zim", "offline", "kolibri"]
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.12"
 description = "Make ZIM file from Kolibri Channels"
 readme = "README.md"
 license = { text = "GPL-3.0-or-later" }
@@ -68,6 +68,9 @@ exclude = ["/.github"]
 path = "hatch_build.py"
 dependencies = ["zimscraperlib==3.1.1"]
 
+[tool.hatch.build.targets.wheel]
+packages = ["src/kolibri2zim"]
+
 [tool.hatch.envs.default]
 features = ["dev"]
 

src/kolibri2zim/entrypoint.py

@@ -5,9 +5,10 @@
 import sys
 
 from kolibri2zim.constants import NAME, SCRAPER, Global, get_logger, set_debug
+from kolibri2zim.scraper import Kolibri2Zim
 
 
-def main():
+def parse_args(raw_args):
     parser = argparse.ArgumentParser(
         prog=NAME,
         description="Scraper to create ZIM files from Kolibri channels",
@@ -37,6 +38,13 @@ def main():
         required=True,
     )
 
+    parser.add_argument(
+        "--lang",
+        help="ZIM Language, used in metadata (should be a ISO-639-3 language code). "
+        "If unspecified, scraper will use 'eng'",
+        default="eng",
+    )
+
     parser.add_argument(
         "--title",
         help="Custom title for your ZIM. Kolibri channel name otherwise",
@@ -79,7 +87,8 @@ def main():
     )
 
     parser.add_argument(
-        "--publisher", help="Custom publisher name (ZIM metadata). “OpenZIM” otherwise"
+        "--publisher",
+        help="Custom publisher name (ZIM metadata). “openZIM” otherwise",
     )
 
     parser.add_argument(
@@ -192,13 +201,14 @@ def main():
         action="version",
         version=SCRAPER,
     )
+    return parser.parse_args(raw_args)
 
-    args = parser.parse_args()
+
+def main():
+    args = parse_args(sys.argv[1:])
     set_debug(args.debug)
     logger = get_logger()
 
-    from kolibri2zim.scraper import Kolibri2Zim
-
     try:
         scraper = Kolibri2Zim(**dict(args._get_kwargs()))
         sys.exit(scraper.run())

src/kolibri2zim/scraper.py

@@ -120,6 +120,7 @@ def go(option):
         self.author = go("creator")
         self.publisher = go("publisher")
         self.name = go("name")
+        self.language = go("lang")
 
         # customization
         self.favicon = go("favicon")
@@ -784,6 +785,7 @@ def run(self):
             f"  description: {self.description}\n"
             f"  creator: {self.author}\n"
             f"  publisher: {self.publisher}\n"
+            f"  language: {self.language}\n"
             f"  tags: {';'.join(self.tags)}"
         )
 
@@ -815,8 +817,8 @@ def run(self):
             ignore_duplicates=True,
         )
         self.creator.config_metadata(
-            Name=self.clean_fname,
-            Language="eng",
+            Name=self.name,  # pyright: ignore reportGeneralTypeIssues
+            Language=self.language,  # pyright: ignore reportGeneralTypeIssues
             Title=self.title,
             Description=self.description,
             LongDescription=self.long_description,
@@ -966,7 +968,7 @@ def sanitize_inputs(self):
         self.author = self.author.strip()
 
         if not self.publisher:
-            self.publisher = "Openzim"
+            self.publisher = "openZIM"
         self.publisher = self.publisher.strip()
 
         self.tags = list({*self.tags, "_category:other", "kolibri", "_videos:yes"})

tests/conftest.py

@@ -6,9 +6,6 @@
 from kolibri2zim.scraper import Kolibri2Zim, KolibriDB
 from kolibri2zim.scraper import options as expected_options_keys
 
-CHANNEL_NAME = "channel_name"
-CHANNEL_DESCRIPTION = "a description"
-
 
 class FakeDb(KolibriDB):
     def __init__(
@@ -30,10 +27,32 @@ def get_channel_metadata(self, _):
 
 
 @pytest.fixture()
-def scraper_generator() -> Generator[Callable[..., Kolibri2Zim], None, None]:
+def channel_name() -> Generator[str, None, None]:
+    yield "channel name"
+
+
+@pytest.fixture()
+def channel_description() -> Generator[str, None, None]:
+    yield "channel description"
+
+
+@pytest.fixture()
+def channel_author() -> Generator[str, None, None]:
+    yield "channel author"
+
+
+@pytest.fixture()
+def zim_name() -> Generator[str, None, None]:
+    yield "a_name"
+
+
+@pytest.fixture()
+def scraper_generator(
+    channel_name, channel_description
+) -> Generator[Callable[..., Kolibri2Zim], None, None]:
     def _scraper(
-        channel_name: str = CHANNEL_NAME,
-        channel_description: str = CHANNEL_DESCRIPTION,
+        channel_name: str = channel_name,
+        channel_description: str = channel_description,
         channel_author: str | None = None,
         additional_options: dict[str, Any] | None = None,
     ) -> Kolibri2Zim:

tests/test_sanitize_inputs.py

@@ -1,13 +1,16 @@
 import random
+import re
 import string
 from collections.abc import Callable
 
 import pytest
+from conftest import FakeDb
 from zimscraperlib.constants import MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LEN
 from zimscraperlib.constants import (
     MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LEN,
 )
 
+from kolibri2zim.entrypoint import parse_args
 from kolibri2zim.scraper import Kolibri2Zim
 
 
@@ -191,3 +194,31 @@ def test_description(
 
     assert scraper.description == expected_description
     assert scraper.long_description == expected_long_description
+
+
+def test_no_required_args():
+    with pytest.raises(expected_exception=SystemExit):
+        parse_args([])
+
+
+def test_defaults_args(channel_name, channel_description, channel_author, zim_name):
+    args = parse_args(["--name", zim_name])
+    scraper = Kolibri2Zim(**dict(args._get_kwargs()))
+    scraper.db = FakeDb(
+        channel_name=channel_name,
+        channel_description=channel_description,
+        channel_author=channel_author,
+    )
+    scraper.sanitize_inputs()
+    assert scraper.language == "eng"
+    assert scraper.publisher == "openZIM"
+    assert scraper.author == channel_author
+    assert scraper.title == channel_name
+    assert scraper.description == channel_description
+    assert scraper.name == zim_name
+    assert re.match(
+        pattern=f"{zim_name}_\\d{{4}}-\\d{{2}}\\.zim", string=scraper.clean_fname
+    )
+    # We compare sets because ordering does not matter
+    assert set(scraper.tags) == {"_category:other", "kolibri", "_videos:yes"}
+    assert len(scraper.tags) == 3